From dbc300038d4e4c833e4ec657e6e9aa7f3c9569bf Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 16 Aug 2024 17:08:39 +0300 Subject: [PATCH 001/120] Improve resolution of callees --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index d1fbf410e..91cfdb893 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -45,10 +45,29 @@ class EtsApplicationGraphImpl( override fun callees(node: EtsStmt): Sequence { val expr = node.callExpr ?: return emptySequence() val callee = expr.method - return cp.classes + val allMethods = cp.classes .asSequence() - .flatMap { it.methods } - .filter { it.signature == callee } + .flatMap { it.methods + it.ctor } + .toList() + + val methodsWithSameName = allMethods.filter { + it.name == callee.name + } + if (methodsWithSameName.size == 1) { + return sequenceOf(methodsWithSameName.first()) + } + + val methodsWithSameClassName = methodsWithSameName.filter { + it.enclosingClass.name == callee.enclosingClass.name + } + if (methodsWithSameClassName.size == 1) { + return sequenceOf(methodsWithSameClassName.first()) + } + + // Else, return all methods with the same signature. + return allMethods.asSequence().filter { + it.signature == callee + } } override fun callers(method: EtsMethod): Sequence { From c4a3e1570d60a4921d1186c18c9f386d05742803 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 19 Aug 2024 14:46:54 +0300 Subject: [PATCH 002/120] Fix test --- .../src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt index 45d711599..09bbd6bf6 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt @@ -27,6 +27,7 @@ import org.jacodb.ets.test.utils.loadEtsFileFromResource import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertIs +import kotlin.test.assertTrue private val logger = mu.KotlinLogging.logger {} @@ -62,11 +63,11 @@ class EtsFileTest { cls.methods.forEach { method -> when (method.name) { "add" -> { - assertEquals(9, method.cfg.instructions.size) + assertTrue( method.cfg.instructions.size > 2) } "main" -> { - assertEquals(4, method.cfg.instructions.size) + assertTrue(method.cfg.instructions.size > 2) } } } From 8379510d1a9b509b8e227a5b51dddc9128ef5c47 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 19 Aug 2024 14:47:14 +0300 Subject: [PATCH 003/120] Fix complex expr (cast) in assignment --- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 477309abc..a118c5a21 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -153,15 +153,22 @@ class EtsMethodBuilder( return etsMethod } + private fun ensureLocal(entity: EtsEntity): EtsLocal { + if (entity is EtsLocal) { + return entity + } + val newLocal = newTempLocal(entity.type) + currentStmts += EtsAssignStmt( + location = loc(), + lhv = newLocal, + rhv = entity, + ) + return newLocal + } + private fun ensureOneAddress(entity: EtsEntity): EtsValue { if (entity is EtsExpr || entity is EtsFieldRef || entity is EtsArrayAccess) { - val newLocal = newTempLocal(entity.type) - currentStmts += EtsAssignStmt( - location = loc(), - lhv = newLocal, - rhv = entity, - ) - return newLocal + return ensureLocal(entity) } else { check(entity is EtsValue) { "Expected EtsValue, but got $entity" @@ -193,13 +200,17 @@ class EtsMethodBuilder( check(lhv is EtsLocal || lhv is EtsFieldRef || lhv is EtsArrayAccess) { "LHV of AssignStmt should be EtsLocal, EtsFieldRef, or EtsArrayAccess, but got $lhv" } - val rhv = convertToEtsEntity(stmt.right).let { + val rhv = convertToEtsEntity(stmt.right).let { rhv -> if (lhv is EtsLocal) { - it - } else if (it is EtsCastExpr || it is EtsNewExpr) { - it + if (rhv is EtsCastExpr && rhv.arg is EtsExpr) { + EtsCastExpr(ensureLocal(rhv.arg), rhv.type) + } else { + rhv + } + } else if (rhv is EtsCastExpr || rhv is EtsNewExpr) { + rhv } else { - ensureOneAddress(it) + ensureOneAddress(rhv) } } EtsAssignStmt( @@ -413,14 +424,14 @@ class EtsMethodBuilder( instance = convertToEtsEntity(value.instance as LocalDto) as EtsLocal, // safe cast method = convertToEtsMethodSignature(value.method), args = value.args.map { - ensureOneAddress(convertToEtsEntity(it)) + ensureLocal(convertToEtsEntity(it)) }, ) is StaticCallExprDto -> EtsStaticCallExpr( method = convertToEtsMethodSignature(value.method), args = value.args.map { - ensureOneAddress(convertToEtsEntity(it)) + ensureLocal(convertToEtsEntity(it)) }, ) From c9fe6cb55cfd93dc1bc9c0587b4956595547a7e2 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Tue, 20 Aug 2024 14:34:48 +0300 Subject: [PATCH 004/120] Add some samples for type inference --- .../samples/source/typeinfer/cast.ts | 12 +++++++++ .../samples/source/typeinfer/microphone.ts | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 jacodb-ets/src/test/resources/samples/source/typeinfer/cast.ts create mode 100644 jacodb-ets/src/test/resources/samples/source/typeinfer/microphone.ts diff --git a/jacodb-ets/src/test/resources/samples/source/typeinfer/cast.ts b/jacodb-ets/src/test/resources/samples/source/typeinfer/cast.ts new file mode 100644 index 000000000..d43f69569 --- /dev/null +++ b/jacodb-ets/src/test/resources/samples/source/typeinfer/cast.ts @@ -0,0 +1,12 @@ +declare function getData(): any; + +type Data = {} + +function entrypoint() { + let x = getData() as Data; + infer(x); +} + +function infer(arg: any) { + console.log(arg); +} diff --git a/jacodb-ets/src/test/resources/samples/source/typeinfer/microphone.ts b/jacodb-ets/src/test/resources/samples/source/typeinfer/microphone.ts new file mode 100644 index 000000000..e436b5f49 --- /dev/null +++ b/jacodb-ets/src/test/resources/samples/source/typeinfer/microphone.ts @@ -0,0 +1,25 @@ +interface Microphone { + uuid: string +} + +class VirtualMicro implements Microphone { + uuid: string = "virtual_micro_v3" +} + +interface Devices { + microphone: Microphone +} + +class VirtualDevices implements Devices { + microphone: Microphone = new VirtualMicro() +} + +function getMicrophoneUuid(device: Devices): string { + return device.microphone.uuid +} + +function entrypoint() { + let devices = new VirtualDevices() + let uuid = getMicrophoneUuid(devices) + console.log(uuid) +} From 188018c9045ccef9c8fb95b18d0a3ff59fdd6176 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Tue, 20 Aug 2024 14:59:56 +0300 Subject: [PATCH 005/120] Unify resource loading --- .../org/jacodb/ets/test/EtsProjectAnalysis.kt | 17 +++++----- .../org/jacodb/ets/test/utils/Entrypoints.kt | 6 +--- .../org/jacodb/ets/test/utils/LoadEts.kt | 4 +-- .../org/jacodb/ets/test/utils/TaintConfig.kt | 4 +-- .../kotlin/org/jacodb/ets/test/utils/Utils.kt | 33 +++++++++++++++++++ 5 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt index e999da8ce..f39779040 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt @@ -30,11 +30,13 @@ import org.jacodb.ets.graph.EtsApplicationGraphImpl import org.jacodb.ets.model.EtsFile import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.test.utils.getResourcePath +import org.jacodb.ets.test.utils.getResourceStream import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIf import java.nio.file.Files -import java.nio.file.Paths import kotlin.io.path.exists +import kotlin.io.path.toPath import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds @@ -60,10 +62,8 @@ class EtsProjectAnalysis { } private fun countFileLines(path: String): Long { - val stream = object {}::class.java.getResourceAsStream(path) - ?: error("Resource not found: $path") - stream.bufferedReader().use { reader -> - return reader.lines().count() + return getResourceStream(path).bufferedReader().use { reader -> + reader.lines().count() } } @@ -71,15 +71,14 @@ class EtsProjectAnalysis { } private fun projectAvailable(): Boolean { - val resource = object {}::class.java.getResource(PROJECT_PATH)?.toURI() - return resource != null && Paths.get(resource).exists() + val path = object {}::class.java.getResource(PROJECT_PATH)?.toURI()?.toPath() + return path != null && path.exists() } @EnabledIf("projectAvailable") @Test fun processAllFiles() { - val baseDirUrl = object {}::class.java.getResource(BASE_PATH) - val baseDir = Paths.get(baseDirUrl?.toURI() ?: error("Resource not found")) + val baseDir = getResourcePath(BASE_PATH) Files.walk(baseDir) .filter { it.toString().endsWith(".json") } .map { baseDir.relativize(it).toString().replace("\\", "/").substringBeforeLast('.') } diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Entrypoints.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Entrypoints.kt index d82bb1b61..4f7a8583e 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Entrypoints.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Entrypoints.kt @@ -21,7 +21,6 @@ import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.model.EtsFile import org.jacodb.ets.utils.dumpDot import org.jacodb.ets.utils.render -import org.jacodb.ets.utils.resolveSibling import org.jacodb.ets.utils.toText import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.Path @@ -29,7 +28,6 @@ import kotlin.io.path.div import kotlin.io.path.name import kotlin.io.path.nameWithoutExtension import kotlin.io.path.relativeTo -import kotlin.io.path.toPath import kotlin.io.path.walk private val logger = mu.KotlinLogging.logger {} @@ -92,9 +90,7 @@ object DumpEtsFilesToDot { @JvmStatic fun main(args: Array) { - val res = ETSIR - val etsirDir = object {}::class.java.getResource(res)?.toURI()?.toPath() - ?: error("Resource not found: '$res'") + val etsirDir = getResourcePath(ETSIR) logger.info { "etsirDir = $etsirDir" } etsirDir.walk() diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt index 05bbabf83..4912f36d3 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt @@ -24,10 +24,8 @@ import org.jacodb.ets.model.EtsFile private val logger = KotlinLogging.logger {} fun loadEtsFileDtoFromResource(jsonPath: String): EtsFileDto { - require(jsonPath.startsWith("/")) { "Resource path must start with '/': $jsonPath" } logger.debug { "Loading EtsIR from resource: '$jsonPath'" } - val stream = object {}::class.java.getResourceAsStream(jsonPath) - ?: error("Resource not found: $jsonPath") + val stream = getResourceStream(jsonPath) return EtsFileDto.loadFromJson(stream) } diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TaintConfig.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TaintConfig.kt index ac18fd3d6..29ad28cf2 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TaintConfig.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TaintConfig.kt @@ -46,9 +46,7 @@ private val json = Json { } fun loadRules(configFileName: String): List { - val configResource = object {}::class.java.getResourceAsStream("/$configFileName") - ?: error("Could not load config from '$configFileName'") - val configJson = configResource.bufferedReader().readText() + val configJson = getResourceStream("/$configFileName").bufferedReader().readText() val rules: List = json.decodeFromString(configJson) // println("Loaded ${rules.size} rules from '$configFileName'") // for (rule in rules) { diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt new file mode 100644 index 000000000..365a9f324 --- /dev/null +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.ets.test.utils + +import java.io.InputStream +import java.nio.file.Path +import kotlin.io.path.toPath + +fun getResourcePath(res: String): Path { + require(res.startsWith("/")) { "Resource path must start with '/': '$res'" } + return object {}::class.java.getResource(res)?.toURI()?.toPath() + ?: error("Resource not found: '$res'") +} + +fun getResourceStream(res: String): InputStream { + require(res.startsWith("/")) { "Resource path must start with '/': '$res'" } + return object {}::class.java.getResourceAsStream(res) + ?: error("Resource not found: '$res'") +} From e3c12928149e042ce086888d951da8280155b14b Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 21 Aug 2024 16:24:38 +0300 Subject: [PATCH 006/120] Lazy callees --- .../kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 91cfdb893..957f21f29 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -47,25 +47,24 @@ class EtsApplicationGraphImpl( val callee = expr.method val allMethods = cp.classes .asSequence() - .flatMap { it.methods + it.ctor } - .toList() + .flatMap { it.methods.asSequence() + it.ctor } val methodsWithSameName = allMethods.filter { it.name == callee.name } - if (methodsWithSameName.size == 1) { + if (methodsWithSameName.count() == 1) { return sequenceOf(methodsWithSameName.first()) } val methodsWithSameClassName = methodsWithSameName.filter { it.enclosingClass.name == callee.enclosingClass.name } - if (methodsWithSameClassName.size == 1) { + if (methodsWithSameClassName.count() == 1) { return sequenceOf(methodsWithSameClassName.first()) } // Else, return all methods with the same signature. - return allMethods.asSequence().filter { + return allMethods.filter { it.signature == callee } } From 79b2fae318b09585600823404ea9c8be0e8f0217 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 28 Aug 2024 14:41:13 +0300 Subject: [PATCH 007/120] Fix new array element type --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index a118c5a21..3f9a23f91 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -304,7 +304,7 @@ class EtsMethodBuilder( ) is NewArrayExprDto -> EtsNewArrayExpr( - elementType = convertToEtsType(value.type), + elementType = convertToEtsType(value.elementType), size = convertToEtsEntity(value.size), ) From 78c049d8cc05675ca9dc5cc93b8dd2d97a71ff38 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 28 Aug 2024 14:41:41 +0300 Subject: [PATCH 008/120] Add enclosingFile property --- .../src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt index 927f44d21..e91d0a566 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt @@ -46,6 +46,9 @@ data class EtsNamespaceSignature( val file: EtsFileSignature? = null, val namespace: EtsNamespaceSignature? = null, ) { + val enclosingFile: EtsFileSignature? + get() = file ?: namespace?.enclosingFile + override fun toString(): String { // TODO: 'file' is not included in the toString() output, // because it only disturbs the debugging output. @@ -71,6 +74,9 @@ data class EtsClassSignature( // } // } + val enclosingFile: EtsFileSignature? + get() = file ?: namespace?.enclosingFile + override fun toString(): String { return if (namespace != null) { "$namespace::$name" From ea7e05b875f95e32e641e6f81b066126d3ad65ba Mon Sep 17 00:00:00 2001 From: Vladislav Kasimov Date: Wed, 4 Sep 2024 14:32:01 +0300 Subject: [PATCH 009/120] Implement basic method lookup --- .../kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 957f21f29..e5b5442f5 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -45,6 +45,15 @@ class EtsApplicationGraphImpl( override fun callees(node: EtsStmt): Sequence { val expr = node.callExpr ?: return emptySequence() val callee = expr.method + + // TODO: Fix later + if (callee.enclosingClass.name.isBlank()) { + val clazz = cp.classes.single { it.signature == node.method.enclosingClass } + return clazz.methods.asSequence().filter { it.name == callee.name } + } else { + check(callee.name == "@instance_init" || callee.name == "constructor") + } + val allMethods = cp.classes .asSequence() .flatMap { it.methods.asSequence() + it.ctor } From de07c72cef09645edf824a5cd124cc6098bfa257 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Tue, 10 Sep 2024 16:19:36 +0300 Subject: [PATCH 010/120] Improve callees resolve --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 63 ++++++++++++------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index e5b5442f5..49b6c0a40 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -18,6 +18,8 @@ package org.jacodb.ets.graph import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.ets.base.EtsStmt +import org.jacodb.ets.model.EtsClassSignature +import org.jacodb.ets.model.EtsFileSignature import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsScene import org.jacodb.ets.utils.callExpr @@ -26,6 +28,38 @@ interface EtsApplicationGraph : ApplicationGraph { val cp: EtsScene } +private fun EtsFileSignature?.isUnknown(): Boolean = + this == null || fileName.isBlank() || fileName == "_UnknownFileName" + +private fun EtsClassSignature.isUnknown(): Boolean = + name.isBlank() + +enum class ComparisonResult { + Equal, + NotEqual, + Unknown, +} + +fun compareFileSignatures( + sig1: EtsFileSignature?, + sig2: EtsFileSignature?, +): ComparisonResult = when { + sig1.isUnknown() -> ComparisonResult.Unknown + sig2.isUnknown() -> ComparisonResult.Unknown + sig1?.fileName == sig2?.fileName -> ComparisonResult.Equal + else -> ComparisonResult.NotEqual +} + +fun compareClassSignatures( + sig1: EtsClassSignature, + sig2: EtsClassSignature, +): ComparisonResult = when { + sig1.isUnknown() -> ComparisonResult.Unknown + sig2.isUnknown() -> ComparisonResult.Unknown + sig1.name == sig2.name -> compareFileSignatures(sig1.file, sig2.file) + else -> ComparisonResult.NotEqual +} + class EtsApplicationGraphImpl( override val cp: EtsScene, ) : EtsApplicationGraph { @@ -46,36 +80,17 @@ class EtsApplicationGraphImpl( val expr = node.callExpr ?: return emptySequence() val callee = expr.method - // TODO: Fix later + // TODO: hack if (callee.enclosingClass.name.isBlank()) { val clazz = cp.classes.single { it.signature == node.method.enclosingClass } - return clazz.methods.asSequence().filter { it.name == callee.name } - } else { - check(callee.name == "@instance_init" || callee.name == "constructor") + return (clazz.methods.asSequence() + clazz.ctor).filter { it.name == callee.name } } - val allMethods = cp.classes + return cp.classes .asSequence() + .filter { compareClassSignatures(it.signature, callee.enclosingClass) != ComparisonResult.NotEqual } .flatMap { it.methods.asSequence() + it.ctor } - - val methodsWithSameName = allMethods.filter { - it.name == callee.name - } - if (methodsWithSameName.count() == 1) { - return sequenceOf(methodsWithSameName.first()) - } - - val methodsWithSameClassName = methodsWithSameName.filter { - it.enclosingClass.name == callee.enclosingClass.name - } - if (methodsWithSameClassName.count() == 1) { - return sequenceOf(methodsWithSameClassName.first()) - } - - // Else, return all methods with the same signature. - return allMethods.filter { - it.signature == callee - } + .filter { it.signature.name == callee.name } } override fun callers(method: EtsMethod): Sequence { From 1dd300313c9300bc14730d19cdb4c5c5285df70d Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Tue, 10 Sep 2024 16:20:10 +0300 Subject: [PATCH 011/120] Add sample with scoped local --- .../src/test/resources/samples/source/lang/scoped.ts | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 jacodb-ets/src/test/resources/samples/source/lang/scoped.ts diff --git a/jacodb-ets/src/test/resources/samples/source/lang/scoped.ts b/jacodb-ets/src/test/resources/samples/source/lang/scoped.ts new file mode 100644 index 000000000..4f5eaa0e8 --- /dev/null +++ b/jacodb-ets/src/test/resources/samples/source/lang/scoped.ts @@ -0,0 +1,8 @@ +function main() { + let x = 42; + if (true) { + let x = "kek"; + console.log(x); + } + console.log(x); +} From cda4f41e52837bc529f557f8170f6e61d0e38f2f Mon Sep 17 00:00:00 2001 From: MForest7 Date: Fri, 13 Sep 2024 12:48:20 +0300 Subject: [PATCH 012/120] mark first block as visited --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 3f9a23f91..156205036 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -475,7 +475,7 @@ class EtsMethodBuilder( "Method body should contain at least return stmt" } - val visited: MutableSet = hashSetOf() + val visited: MutableSet = hashSetOf(0) val queue: ArrayDeque = ArrayDeque() queue.add(0) From a767a8659f9b4bdfec98ff0be48a9a43b5d7c686 Mon Sep 17 00:00:00 2001 From: Vladislav Kasimov Date: Wed, 18 Sep 2024 15:20:44 +0300 Subject: [PATCH 013/120] Fix literal type --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index 16ff8041d..7a89dfeb3 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -20,6 +20,7 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator +import kotlinx.serialization.json.JsonElement @Serializable @OptIn(ExperimentalSerializationApi::class) @@ -141,13 +142,13 @@ object UndefinedTypeDto : PrimitiveTypeDto { @Serializable @SerialName("LiteralType") data class LiteralTypeDto( - val literal: String, + val literal: JsonElement, ) : PrimitiveTypeDto { override val name: String - get() = "literal" + get() = literal.toString() override fun toString(): String { - return literal + return name } } From eaf386949a0a5a47b109838f0892a2106dc59b21 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 18 Sep 2024 20:49:47 +0300 Subject: [PATCH 014/120] For unknown class signature, do not blindly go into outer constructor and do not go recursively in the same method --- .../src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 49b6c0a40..67410d0d2 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -83,7 +83,7 @@ class EtsApplicationGraphImpl( // TODO: hack if (callee.enclosingClass.name.isBlank()) { val clazz = cp.classes.single { it.signature == node.method.enclosingClass } - return (clazz.methods.asSequence() + clazz.ctor).filter { it.name == callee.name } + return clazz.methods.asSequence().filter { it.name == callee.name }.filterNot { it.name == node.method.name } } return cp.classes From fc8e80425de9636b99c62b147987690bf1340cbc Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 19 Sep 2024 10:56:05 +0300 Subject: [PATCH 015/120] Fix convert for literal type --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 156205036..5d5b3a9cf 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -610,7 +610,7 @@ fun convertToEtsType(type: TypeDto): EtsType { BooleanTypeDto -> EtsBooleanType is LiteralTypeDto -> EtsLiteralType( - literalTypeName = type.literal + literalTypeName = type.literal.toString() ) NullTypeDto -> EtsNullType From eee07ff9910622c2b443dc2db26d69f25e786a2f Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 20 Sep 2024 11:29:40 +0300 Subject: [PATCH 016/120] Yet another take on callees resolver --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 67410d0d2..5c26dec05 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -80,17 +80,40 @@ class EtsApplicationGraphImpl( val expr = node.callExpr ?: return emptySequence() val callee = expr.method - // TODO: hack - if (callee.enclosingClass.name.isBlank()) { - val clazz = cp.classes.single { it.signature == node.method.enclosingClass } - return clazz.methods.asSequence().filter { it.name == callee.name }.filterNot { it.name == node.method.name } + // First, try to resolve the callee via a complete signature match: + val resolvedCompletely = cp.classes + .asSequence() + .filter { compareClassSignatures(it.signature, callee.enclosingClass) == ComparisonResult.Equal } + // Note: include constructors! + .flatMap { it.methods.asSequence() + it.ctor } + .filter { it.name == callee.name } + if (resolvedCompletely.any()) { + return resolvedCompletely } - return cp.classes + // If the complete signature match failed, + // try to find the unique neighbour (non-recursive) method in the same class: + val resolvedNeighbour = cp.classes + .single { it.signature == node.method.enclosingClass } + .methods + .filter { it.name == callee.name } + if (resolvedNeighbour.isNotEmpty()) return resolvedNeighbour.asSequence() + + // If the neighbour match failed, + // try to *uniquely* resolve the callee via a partial signature match: + val resolvedPartially = cp.classes .asSequence() .filter { compareClassSignatures(it.signature, callee.enclosingClass) != ComparisonResult.NotEqual } - .flatMap { it.methods.asSequence() + it.ctor } - .filter { it.signature.name == callee.name } + // Note: omit constructors! + .flatMap { it.methods.asSequence() } + .filter { it.name == callee.name } + // Note: exclude recursive calls: + .filterNot { it.name == node.method.name } + + if (resolvedPartially.none()) return emptySequence() + val resolved = resolvedPartially.toList() + if (resolved.size == 1) return resolved.asSequence() + return emptySequence() } override fun callers(method: EtsMethod): Sequence { From dda8bd77c5f8671ab44e0e9e1366c12d49d65179 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 20 Sep 2024 11:34:00 +0300 Subject: [PATCH 017/120] Exclude recursive callee in neighbor lookup --- .../src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 5c26dec05..77613b210 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -97,6 +97,7 @@ class EtsApplicationGraphImpl( .single { it.signature == node.method.enclosingClass } .methods .filter { it.name == callee.name } + .filterNot { it.name == node.method.name } if (resolvedNeighbour.isNotEmpty()) return resolvedNeighbour.asSequence() // If the neighbour match failed, From f110aa48cfaa7c4ce37decf2d79ef1e964a41666 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 20 Sep 2024 11:34:22 +0300 Subject: [PATCH 018/120] Use sequence --- .../main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 77613b210..6c6b2fff6 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -96,9 +96,10 @@ class EtsApplicationGraphImpl( val resolvedNeighbour = cp.classes .single { it.signature == node.method.enclosingClass } .methods + .asSequence() .filter { it.name == callee.name } .filterNot { it.name == node.method.name } - if (resolvedNeighbour.isNotEmpty()) return resolvedNeighbour.asSequence() + if (resolvedNeighbour.any()) return resolvedNeighbour // If the neighbour match failed, // try to *uniquely* resolve the callee via a partial signature match: From 0a40ef1aba86e7432700c9bd7d35557ed005a9c0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 20 Sep 2024 12:37:46 +0300 Subject: [PATCH 019/120] Cleanup --- .../main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 6c6b2fff6..7550fe7b3 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -87,9 +87,7 @@ class EtsApplicationGraphImpl( // Note: include constructors! .flatMap { it.methods.asSequence() + it.ctor } .filter { it.name == callee.name } - if (resolvedCompletely.any()) { - return resolvedCompletely - } + if (resolvedCompletely.any()) return resolvedCompletely // If the complete signature match failed, // try to find the unique neighbour (non-recursive) method in the same class: From 9e9e03f4f166a32da730c985fc8aad72b790bfbb Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 20 Sep 2024 12:48:27 +0300 Subject: [PATCH 020/120] Add extra check for current method signature --- .../main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 7550fe7b3..fb6156341 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -80,6 +80,11 @@ class EtsApplicationGraphImpl( val expr = node.callExpr ?: return emptySequence() val callee = expr.method + // Note: the resolving code below expects that at least the current method signature is known. + check(!node.method.enclosingClass.isUnknown()) { + "Unknown class signature in method: ${node.method}" + } + // First, try to resolve the callee via a complete signature match: val resolvedCompletely = cp.classes .asSequence() From 15bc33888235d96696bd9c95bb14f754f2d2d2d8 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 20 Sep 2024 13:52:28 +0300 Subject: [PATCH 021/120] Add ad-hoc constructor resolution --- .../org/jacodb/ets/graph/EtsApplicationGraph.kt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index fb6156341..21fadf158 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -85,6 +85,23 @@ class EtsApplicationGraphImpl( "Unknown class signature in method: ${node.method}" } + // Note: specific resolve for constructor: + if (callee.name == "constructor") { + val resolvedCompletely = cp.classes + .asSequence() + .filter { it.signature == callee.enclosingClass } + .map { it.ctor } + if (resolvedCompletely.any()) return resolvedCompletely + + val resolvedPartially = cp.classes + .asSequence() + .filter { it.signature.name == callee.enclosingClass.name } + .map { it.ctor } + .toList() + if (resolvedPartially.size == 1) return resolvedPartially.asSequence() + return emptySequence() + } + // First, try to resolve the callee via a complete signature match: val resolvedCompletely = cp.classes .asSequence() From 8fda1e12587b40d5b6e4b430d4672880f178f35e Mon Sep 17 00:00:00 2001 From: Vladislav Kasimov Date: Fri, 20 Sep 2024 18:16:39 +0300 Subject: [PATCH 022/120] Chane unknown value to string --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt index 10da148d5..07d3439a2 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt @@ -32,7 +32,7 @@ sealed interface ValueDto { @Serializable @SerialName("UNKNOWN_VALUE") data class UnknownValueDto( - val value: JsonElement, + val value: String, ) : ValueDto { override val type: TypeDto get() = UnknownTypeDto From 6cc1dc2ba60292c92b81523e3de5a27d4ff062c7 Mon Sep 17 00:00:00 2001 From: Vladislav Kasimov Date: Mon, 23 Sep 2024 15:40:36 +0300 Subject: [PATCH 023/120] Fix unknown call expr --- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 13 ++++++++++++- .../main/kotlin/org/jacodb/ets/dto/Values.kt | 17 +++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 5d5b3a9cf..a89bd07da 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -221,6 +221,17 @@ class EtsMethodBuilder( } is CallStmtDto -> { + if (stmt.expr is UnknownValueDto) { + return object : EtsStmt { + override val location: EtsInstLocation = loc() + + override fun toString(): String = "UnknownCall(${stmt.expr})" + + override fun accept(visitor: EtsStmt.Visitor): R { + error("UnknownCall is not supported") + } + } + } val expr = convertToEtsEntity(stmt.expr) as EtsCallExpr EtsCallStmt( location = loc(), @@ -282,7 +293,7 @@ class EtsMethodBuilder( is UnknownValueDto -> object : EtsEntity { override val type: EtsType = EtsUnknownType - override fun toString(): String = "Unknown(${value.value})" + override fun toString(): String = "UnknownValue(${value.value})" override fun accept(visitor: EtsEntity.Visitor): R { if (visitor is EtsEntity.Visitor.Default) { diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt index 07d3439a2..98b60d0e6 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt @@ -20,7 +20,6 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator -import kotlinx.serialization.json.JsonElement @Serializable @OptIn(ExperimentalSerializationApi::class) @@ -33,12 +32,18 @@ sealed interface ValueDto { @SerialName("UNKNOWN_VALUE") data class UnknownValueDto( val value: String, -) : ValueDto { +) : ValueDto, CallExprDto { override val type: TypeDto get() = UnknownTypeDto + override val args: List? + get() = null + + override val method: MethodSignatureDto? + get() = null + override fun toString(): String { - return "UNKNOWN($value)" + return value } } @@ -338,11 +343,11 @@ data class RelationOperationDto( @Serializable sealed interface CallExprDto : ExprDto { - val method: MethodSignatureDto - val args: List + val method: MethodSignatureDto? + val args: List? override val type: TypeDto - get() = method.returnType + get() = method?.returnType ?: UnknownTypeDto } @Serializable From c3b7595570a8881dd203368726a36acd9d862d75 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 14:05:28 +0300 Subject: [PATCH 024/120] Fix path for local project --- .../test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt index f39779040..633534a09 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt @@ -51,10 +51,10 @@ class EtsProjectAnalysis { private var totalSinks: MutableList> = mutableListOf() companion object : EtsTraits { - private const val SOURCE_PROJECT_PATH = "/project1" - private const val PROJECT_PATH = "/etsir/project1" + private const val SOURCE_PROJECT_PATH = "/projects/applications_app_samples/source/applications_app_samples/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc" + private const val PROJECT_PATH = "/projects/applications_app_samples/etsir/ast/ArkTSDistributedCalc" private const val START_PATH = "/entry/src/main/ets" - private const val BASE_PATH = PROJECT_PATH + START_PATH + private const val BASE_PATH = PROJECT_PATH private const val SOURCE_BASE_PATH = SOURCE_PROJECT_PATH + START_PATH private fun loadFromProject(filename: String): EtsFile { From 6202d2c2eb213fd313a4e45da5366b95b010c30a Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 15:57:56 +0300 Subject: [PATCH 025/120] Cleanup --- jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt | 4 ++-- .../src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt index 2d43715b9..22be9003d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt @@ -18,9 +18,9 @@ package org.jacodb.ets.model import org.jacodb.api.common.CommonProject -class EtsScene ( +class EtsScene( val files: List, -): CommonProject { +) : CommonProject { val classes: List get() = files.flatMap { it.classes } } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index 4baf176a8..cb6a057cf 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -23,13 +23,10 @@ import java.io.FileNotFoundException import java.nio.file.Path import kotlin.io.path.Path import kotlin.io.path.absolute -import kotlin.io.path.createTempDirectory -import kotlin.io.path.div import kotlin.io.path.exists import kotlin.io.path.inputStream import kotlin.io.path.nameWithoutExtension import kotlin.io.path.pathString -import kotlin.io.path.relativeTo import kotlin.time.Duration.Companion.seconds private const val ENV_VAR_ARK_ANALYZER_DIR = "ARKANALYZER_DIR" From 8023bd6650c2bd4315c46cfa023a52598e558f82 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 14:06:11 +0300 Subject: [PATCH 026/120] Update model for AA/neo/2024-09-23 --- .github/workflows/build-and-test.yml | 2 +- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt | 1 - jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt | 1 - .../src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 18e4c7367..f56a4fe44 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-08-16" + BRANCH="neo/2024-09-23" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt index 80c85c3ac..57e7f735b 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt @@ -65,7 +65,6 @@ data class ClassDto( @Serializable data class FieldDto( val signature: FieldSignatureDto, - val typeParameters: List, val modifiers: List? = null, @SerialName("questionToken") val isOptional: Boolean = false, // '?' @SerialName("exclamationToken") val isDefinitelyAssigned: Boolean = false, // '!' diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt index d6be175a7..e313c33fd 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt @@ -75,7 +75,6 @@ fun EtsFileDto.toText(): String { lines += " fields: ${clazz.fields.size}" clazz.fields.forEach { field -> lines += " - FIELD '${field.signature}'" - lines += " typeParameters = ${field.typeParameters}" lines += " modifiers = ${field.modifiers}" lines += " isOptional = ${field.isOptional}" lines += " isDefinitelyAssigned = ${field.isDefinitelyAssigned}" diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 68c0cb2c0..c2c904e9c 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -112,7 +112,6 @@ class EtsFromJsonTest { type = NumberTypeDto, ), modifiers = emptyList(), - typeParameters = emptyList(), isOptional = true, isDefinitelyAssigned = false, ) From 4c291b0832f309350f35a69d73f4702546b6eeb4 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 16:43:21 +0300 Subject: [PATCH 027/120] Revert typeParameters in FieldDto to support deserializing outdated DTOs --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt index 57e7f735b..18f9117af 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt @@ -65,6 +65,7 @@ data class ClassDto( @Serializable data class FieldDto( val signature: FieldSignatureDto, + val typeParameters: List? = null, // TODO: remove val modifiers: List? = null, @SerialName("questionToken") val isOptional: Boolean = false, // '?' @SerialName("exclamationToken") val isDefinitelyAssigned: Boolean = false, // '!' From f9667fb4b8f3561712ba0478eed622f1d28b6874 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 16:47:23 +0300 Subject: [PATCH 028/120] Add PtrCallExpr, fix unknown values/stmts --- .../kotlin/org/jacodb/ets/base/EtsExpr.kt | 16 +++++++++++ .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 22 +++++++-------- .../main/kotlin/org/jacodb/ets/dto/Stmts.kt | 9 ++++-- .../main/kotlin/org/jacodb/ets/dto/Values.kt | 28 +++++++++++-------- .../org/jacodb/ets/utils/GetOperands.kt | 4 +++ 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsExpr.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsExpr.kt index c17cb586f..8751d607e 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsExpr.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsExpr.kt @@ -77,6 +77,7 @@ interface EtsExpr : EtsEntity { // Call fun visit(expr: EtsInstanceCallExpr): R fun visit(expr: EtsStaticCallExpr): R + fun visit(expr: EtsPtrCallExpr): R // Other fun visit(expr: EtsCommaExpr): R @@ -133,6 +134,7 @@ interface EtsExpr : EtsEntity { override fun visit(expr: EtsInstanceCallExpr): R = defaultVisit(expr) override fun visit(expr: EtsStaticCallExpr): R = defaultVisit(expr) + override fun visit(expr: EtsPtrCallExpr): R = defaultVisit(expr) override fun visit(expr: EtsCommaExpr): R = defaultVisit(expr) override fun visit(expr: EtsTernaryExpr): R = defaultVisit(expr) @@ -790,6 +792,20 @@ data class EtsStaticCallExpr( } } +data class EtsPtrCallExpr( + val ptr: EtsLocal, + override val method: EtsMethodSignature, + override val args: List, +) : EtsCallExpr { + override fun toString(): String { + return "${ptr}.${method.name}(${args.joinToString()})" + } + + override fun accept(visitor: EtsExpr.Visitor): R { + return visitor.visit(this) + } +} + data class EtsCommaExpr( override val left: EtsEntity, override val right: EtsEntity, diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index a89bd07da..1e8860fd6 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -77,6 +77,7 @@ import org.jacodb.ets.base.EtsOrExpr import org.jacodb.ets.base.EtsParameterRef import org.jacodb.ets.base.EtsPreDecExpr import org.jacodb.ets.base.EtsPreIncExpr +import org.jacodb.ets.base.EtsPtrCallExpr import org.jacodb.ets.base.EtsRemExpr import org.jacodb.ets.base.EtsReturnStmt import org.jacodb.ets.base.EtsRightShiftExpr @@ -182,7 +183,7 @@ class EtsMethodBuilder( is UnknownStmtDto -> object : EtsStmt { override val location: EtsInstLocation = loc() - override fun toString(): String = "Unknown(${stmt.stmt})" + override fun toString(): String = "UnknownStmt(${stmt.stmt})" // TODO: equals/hashCode ??? @@ -221,17 +222,6 @@ class EtsMethodBuilder( } is CallStmtDto -> { - if (stmt.expr is UnknownValueDto) { - return object : EtsStmt { - override val location: EtsInstLocation = loc() - - override fun toString(): String = "UnknownCall(${stmt.expr})" - - override fun accept(visitor: EtsStmt.Visitor): R { - error("UnknownCall is not supported") - } - } - } val expr = convertToEtsEntity(stmt.expr) as EtsCallExpr EtsCallStmt( location = loc(), @@ -446,6 +436,14 @@ class EtsMethodBuilder( }, ) + is PtrCallExprDto -> EtsPtrCallExpr( + ptr = convertToEtsEntity(value.ptr as LocalDto) as EtsLocal, // safe cast + method = convertToEtsMethodSignature(value.method), + args = value.args.map { + ensureLocal(convertToEtsEntity(it)) + }, + ) + is ThisRefDto -> EtsThis( type = convertToEtsType(value.type as ClassTypeDto) as EtsClassType // safe cast ) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Stmts.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Stmts.kt index d0e9d4897..0b4cdc77c 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Stmts.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Stmts.kt @@ -20,7 +20,6 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator -import kotlinx.serialization.json.JsonElement @Serializable @OptIn(ExperimentalSerializationApi::class) @@ -30,8 +29,12 @@ sealed interface StmtDto @Serializable @SerialName("UNKNOWN_STMT") data class UnknownStmtDto( - val stmt: JsonElement, -) : StmtDto + val stmt: String, +) : StmtDto { + override fun toString(): String { + return "UNKNOWN($stmt)" + } +} @Serializable @SerialName("NopStmt") diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt index 98b60d0e6..0d8962b6b 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Values.kt @@ -32,18 +32,12 @@ sealed interface ValueDto { @SerialName("UNKNOWN_VALUE") data class UnknownValueDto( val value: String, -) : ValueDto, CallExprDto { +) : ValueDto { override val type: TypeDto get() = UnknownTypeDto - override val args: List? - get() = null - - override val method: MethodSignatureDto? - get() = null - override fun toString(): String { - return value + return "UNKNOWN($value)" } } @@ -343,11 +337,11 @@ data class RelationOperationDto( @Serializable sealed interface CallExprDto : ExprDto { - val method: MethodSignatureDto? - val args: List? + val method: MethodSignatureDto + val args: List override val type: TypeDto - get() = method?.returnType ?: UnknownTypeDto + get() = method.returnType } @Serializable @@ -373,6 +367,18 @@ data class StaticCallExprDto( } } +@Serializable +@SerialName("PtrCallExpr") +data class PtrCallExprDto( + val ptr: ValueDto, // Local + override val method: MethodSignatureDto, + override val args: List, +) : CallExprDto { + override fun toString(): String { + return "$ptr@${method.declaringClass.name}::${method.name}(${args.joinToString()})" + } +} + @Serializable sealed interface RefDto : ValueDto diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/GetOperands.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/GetOperands.kt index 73a54ae30..324cc8db5 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/GetOperands.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/GetOperands.kt @@ -65,6 +65,7 @@ import org.jacodb.ets.base.EtsPostDecExpr import org.jacodb.ets.base.EtsPostIncExpr import org.jacodb.ets.base.EtsPreDecExpr import org.jacodb.ets.base.EtsPreIncExpr +import org.jacodb.ets.base.EtsPtrCallExpr import org.jacodb.ets.base.EtsRemExpr import org.jacodb.ets.base.EtsReturnStmt import org.jacodb.ets.base.EtsRightShiftExpr @@ -295,6 +296,9 @@ private object EntityGetOperands : EtsEntity.Visitor> { override fun visit(expr: EtsStaticCallExpr): Sequence = expr.args.asSequence() + override fun visit(expr: EtsPtrCallExpr): Sequence = + sequenceOf(expr.ptr) + expr.args.asSequence() + override fun visit(expr: EtsCommaExpr): Sequence = sequenceOf(expr.left, expr.right) From 753fd9bc5e681e702457fcc9a0eb70ba0e5178a0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 17:08:16 +0300 Subject: [PATCH 029/120] Improve EtsFile loaders, add comments --- .../org/jacodb/ets/test/utils/LoadEts.kt | 90 ++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt index 4912f36d3..95881d7d1 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt @@ -20,16 +20,102 @@ import mu.KotlinLogging import org.jacodb.ets.dto.EtsFileDto import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.model.EtsFile +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.inputStream +import kotlin.io.path.relativeTo +import kotlin.io.path.walk private val logger = KotlinLogging.logger {} +/** + * Load an [EtsFileDto] from a resource file. + * + * For example, `resources/ets/sample.json` can be loaded with: + * ``` + * val dto: EtsFileDto = loadEtsFileDtoFromResource("/ets/sample.json") + * ``` + */ fun loadEtsFileDtoFromResource(jsonPath: String): EtsFileDto { logger.debug { "Loading EtsIR from resource: '$jsonPath'" } - val stream = getResourceStream(jsonPath) - return EtsFileDto.loadFromJson(stream) + getResourceStream(jsonPath).use { stream -> + return EtsFileDto.loadFromJson(stream) + } } +/** + * Load an [EtsFile] from a resource file. + * + * For example, `resources/ets/sample.json` can be loaded with: + * ``` + * val file: EtsFile = loadEtsFileFromResource("/ets/sample.json") + * ``` + */ fun loadEtsFileFromResource(jsonPath: String): EtsFile { val etsFileDto = loadEtsFileDtoFromResource(jsonPath) return convertToEtsFile(etsFileDto) } + +/** + * Load multiple [EtsFile]s from a resource directory. + * + * For example, all files in `resources/project/` can be loaded with: + * ``` + * val files: Sequence = loadMultipleEtsFilesFromResourceDirectory("/project") + * ``` + */ +@OptIn(ExperimentalPathApi::class) +fun loadMultipleEtsFilesFromResourceDirectory(dirPath: String): Sequence { + val rootPath = getResourcePath(dirPath) + return rootPath.walk().map { path -> + loadEtsFileFromResource("$dirPath/${path.relativeTo(rootPath)}") + } +} + +fun loadMultipleEtsFilesFromMultipleResourceDirectories( + dirPaths: List, +): Sequence { + return dirPaths.asSequence().flatMap { loadMultipleEtsFilesFromResourceDirectory(it) } +} + +//----------------------------------------------------------------------------- + +/** + * Load an [EtsFileDto] from a file. + * + * For example, `data/sample.json` can be loaded with: + * ``` + * val dto: EtsFileDto = loadEtsFileDto("data/sample.json".toPath()) + * ``` + */ +fun loadEtsFileDto(path: Path): EtsFileDto { + path.inputStream().use { stream -> + return EtsFileDto.loadFromJson(stream) + } +} + +/** + * Load an [EtsFile] from a file. + * + * For example, `data/sample.json` can be loaded with: + * ``` + * val file: EtsFile = loadEtsFile("data/sample.json".toPath()) + * ``` + */ +fun loadEtsFile(path: Path): EtsFile { + val etsFileDto = loadEtsFileDto(path) + return convertToEtsFile(etsFileDto) +} + +/** + * Load multiple [EtsFile]s from a directory. + * + * For example, all files in `data` can be loaded with: + * ``` + * val files: Sequence = loadMultipleEtsFilesFromDirectory("data".toPath()) + * ``` + */ +@OptIn(ExperimentalPathApi::class) +fun loadMultipleEtsFilesFromDirectory(dirPath: Path): Sequence { + return dirPath.walk().map { loadEtsFile(it) } +} From 2e3bf99b9a2c3cb16b08a0657e80cbeb948daa12 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 23 Sep 2024 19:18:54 +0300 Subject: [PATCH 030/120] Add test for loading the project --- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index c2c904e9c..96bdb346e 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -40,25 +40,33 @@ import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.dto.convertToEtsMethod import org.jacodb.ets.model.EtsClassSignature import org.jacodb.ets.model.EtsMethodSignature +import org.jacodb.ets.model.EtsScene import org.jacodb.ets.test.utils.loadEtsFileDtoFromResource +import org.jacodb.ets.test.utils.loadMultipleEtsFilesFromResourceDirectory import org.jacodb.ets.utils.loadEtsFileAutoConvert import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIf +import kotlin.io.path.exists import kotlin.io.path.toPath class EtsFromJsonTest { - private val json = Json { - // classDiscriminator = "_" - prettyPrint = true - } + companion object { + private val json = Json { + // classDiscriminator = "_" + prettyPrint = true + } - private val defaultSignature = EtsMethodSignature( - enclosingClass = EtsClassSignature(name = "_DEFAULT_ARK_CLASS"), - name = "_DEFAULT_ARK_METHOD", - parameters = emptyList(), - returnType = EtsAnyType, - ) + private val defaultSignature = EtsMethodSignature( + enclosingClass = EtsClassSignature(name = "_DEFAULT_ARK_CLASS"), + name = "_DEFAULT_ARK_METHOD", + parameters = emptyList(), + returnType = EtsAnyType, + ) + + private const val PROJECT_PATH = "/projects/AppSamples/ArkTSDistributedCalc" + } @Test fun testLoadEtsFileFromJson() { @@ -78,6 +86,28 @@ class EtsFromJsonTest { println("etsFile = $etsFile") } + private fun projectAvailable(): Boolean { + val path = this::class.java.getResource(PROJECT_PATH)?.toURI()?.toPath() + return path != null && path.exists() + } + + @EnabledIf("projectAvailable") + @Test + fun testLoadEtsProject() { + val path = "/projects/AppSamples/ArkTSDistributedCalc" + val files = loadMultipleEtsFilesFromResourceDirectory("$path/etsir").toList() + println("Loaded ${files.size} files") + val project = EtsScene(files) + println("project = $project") + println("Classes: ${project.classes.size}") + for (cls in project.classes) { + println("= ${cls.signature} with ${cls.methods.size} methods:") + for (method in cls.methods) { + println(" - ${method.signature}") + } + } + } + @Test fun testLoadValueFromJson() { val jsonString = """ From fc53c3b73500b4be8a8ed761ea238ea1711a7ddd Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 13:44:13 +0300 Subject: [PATCH 031/120] Add comment for literal type --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index 7a89dfeb3..abfd9beae 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -142,7 +142,7 @@ object UndefinedTypeDto : PrimitiveTypeDto { @Serializable @SerialName("LiteralType") data class LiteralTypeDto( - val literal: JsonElement, + val literal: JsonElement, // string | boolean | number ) : PrimitiveTypeDto { override val name: String get() = literal.toString() From bf7d6dfc17740b92710b93e4322b0ba5dd6e73fe Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 15:06:50 +0300 Subject: [PATCH 032/120] Add script for setting up local projects --- .../src/test/resources/prepare_projects.sh | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 jacodb-ets/src/test/resources/prepare_projects.sh diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh new file mode 100644 index 000000000..16afda3cf --- /dev/null +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -0,0 +1,40 @@ +#/bin/bash +set -euo pipefail + +if [[ -z "${ARKANALYZER_DIR}" ]]; then + echo "ARKANALYZER_DIR is undefined" + exit 1 +fi + +echo "ARKANALYZER_DIR=${ARKANALYZER_DIR}" +SCRIPT=$ARKANALYZER_DIR/src/save/serializeArkIR.ts + +pushd "$(dirname $0)/projects" + +function prepare_app_samples() { + echo + echo "=== Preparing AppSamples..." + echo + mkdir -p AppSamples + pushd AppSamples + + REPO="../../repos/applications_app_samples" + if [[ ! -d $REPO ]]; then + echo "Repository not found: $REPO" + exit 1 + fi + + function prepare_calc() { + NAME="ArkTSDistributedCalc" + echo "= Preparing subproject: $NAME" + mkdir -p $NAME + SRC="$NAME/source" + ETSIR="$NAME/etsir" + ln -srfT "$REPO/code/SuperFeature/DistributedAppDev/$NAME/entry/src/main/ets" $SRC + npx ts-node $SCRIPT -p $SRC $ETSIR -v + } + + prepare_calc +} + +prepare_app_samples From c32f8af1f729d0ee9a10ca627963d9086ba3a324 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 15:07:01 +0300 Subject: [PATCH 033/120] Fix gitignore for samples --- jacodb-ets/src/test/resources/.gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/resources/.gitignore b/jacodb-ets/src/test/resources/.gitignore index 730a420e7..1194cf894 100644 --- a/jacodb-ets/src/test/resources/.gitignore +++ b/jacodb-ets/src/test/resources/.gitignore @@ -1,3 +1,3 @@ -/samples/etsir/ -/samples/abc/ +/samples +!/samples/source /projects From 9470018859a322f7a1b3ae97c518c32d2f5b4858 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 16:03:28 +0300 Subject: [PATCH 034/120] Improve utility functions --- .../kotlin/org/jacodb/ets/test/utils/LoadEts.kt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt index 95881d7d1..0670d447a 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt @@ -22,6 +22,7 @@ import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.model.EtsFile import java.nio.file.Path import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.extension import kotlin.io.path.inputStream import kotlin.io.path.relativeTo import kotlin.io.path.walk @@ -38,6 +39,7 @@ private val logger = KotlinLogging.logger {} */ fun loadEtsFileDtoFromResource(jsonPath: String): EtsFileDto { logger.debug { "Loading EtsIR from resource: '$jsonPath'" } + require(jsonPath.endsWith(".json")) { "File must have a '.json' extension: '$jsonPath'" } getResourceStream(jsonPath).use { stream -> return EtsFileDto.loadFromJson(stream) } @@ -67,7 +69,7 @@ fun loadEtsFileFromResource(jsonPath: String): EtsFile { @OptIn(ExperimentalPathApi::class) fun loadMultipleEtsFilesFromResourceDirectory(dirPath: String): Sequence { val rootPath = getResourcePath(dirPath) - return rootPath.walk().map { path -> + return rootPath.walk().filter { it.extension == "json" }.map { path -> loadEtsFileFromResource("$dirPath/${path.relativeTo(rootPath)}") } } @@ -85,10 +87,11 @@ fun loadMultipleEtsFilesFromMultipleResourceDirectories( * * For example, `data/sample.json` can be loaded with: * ``` - * val dto: EtsFileDto = loadEtsFileDto("data/sample.json".toPath()) + * val dto: EtsFileDto = loadEtsFileDto(Path("data/sample.json")) * ``` */ fun loadEtsFileDto(path: Path): EtsFileDto { + require(path.extension == "json") { "File must have a '.json' extension: $path" } path.inputStream().use { stream -> return EtsFileDto.loadFromJson(stream) } @@ -99,7 +102,7 @@ fun loadEtsFileDto(path: Path): EtsFileDto { * * For example, `data/sample.json` can be loaded with: * ``` - * val file: EtsFile = loadEtsFile("data/sample.json".toPath()) + * val file: EtsFile = loadEtsFile(Path("data/sample.json")) * ``` */ fun loadEtsFile(path: Path): EtsFile { @@ -112,10 +115,10 @@ fun loadEtsFile(path: Path): EtsFile { * * For example, all files in `data` can be loaded with: * ``` - * val files: Sequence = loadMultipleEtsFilesFromDirectory("data".toPath()) + * val files: Sequence = loadMultipleEtsFilesFromDirectory(Path("data")) * ``` */ @OptIn(ExperimentalPathApi::class) fun loadMultipleEtsFilesFromDirectory(dirPath: Path): Sequence { - return dirPath.walk().map { loadEtsFile(it) } + return dirPath.walk().filter { it.extension == "json" }.map { loadEtsFile(it) } } From b509a0a32abbd2a3204b270ea677fa4c59e4b544 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 16:03:51 +0300 Subject: [PATCH 035/120] Ignore repos --- jacodb-ets/src/test/resources/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/src/test/resources/.gitignore b/jacodb-ets/src/test/resources/.gitignore index 1194cf894..1fe1a8b3b 100644 --- a/jacodb-ets/src/test/resources/.gitignore +++ b/jacodb-ets/src/test/resources/.gitignore @@ -1,3 +1,4 @@ /samples !/samples/source /projects +/repos From 6d6666132b64289025638e780f14f2b4e011a2be Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 16:04:07 +0300 Subject: [PATCH 036/120] Prepare AudioPicker project --- .../src/test/resources/prepare_projects.sh | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 16afda3cf..ef20b2128 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -9,20 +9,31 @@ fi echo "ARKANALYZER_DIR=${ARKANALYZER_DIR}" SCRIPT=$ARKANALYZER_DIR/src/save/serializeArkIR.ts -pushd "$(dirname $0)/projects" +pushd "$(dirname $0)" >/dev/null +mkdir -p projects +pushd projects + +function check_repo() { + if [[ ! -d $1 ]]; then + echo "Repository not found: $1" + exit 1 + fi +} function prepare_app_samples() { + NAME="AppSamples" echo - echo "=== Preparing AppSamples..." + echo "=== Preparing $NAME..." echo - mkdir -p AppSamples - pushd AppSamples + if [ -d $NAME ]; then + echo "Directory already exists: $NAME" + return + fi + mkdir -p $NAME + pushd $NAME >/dev/null REPO="../../repos/applications_app_samples" - if [[ ! -d $REPO ]]; then - echo "Repository not found: $REPO" - exit 1 - fi + check_repo $REPO function prepare_calc() { NAME="ArkTSDistributedCalc" @@ -30,7 +41,9 @@ function prepare_app_samples() { mkdir -p $NAME SRC="$NAME/source" ETSIR="$NAME/etsir" + echo "Linking sources..." ln -srfT "$REPO/code/SuperFeature/DistributedAppDev/$NAME/entry/src/main/ets" $SRC + echo "Serializing..." npx ts-node $SCRIPT -p $SRC $ETSIR -v } @@ -38,3 +51,28 @@ function prepare_app_samples() { } prepare_app_samples + +function prepare_audiopicker() { + NAME="AudioPicker" + echo + echo "=== Preparing $NAME..." + echo + if [ -d $NAME ]; then + echo "Directory already exists: $NAME" + return + fi + mkdir -p $NAME + pushd $NAME >/dev/null + + REPO="../../repos/applications_filepicker" + check_repo $REPO + + SRC="source" + ETSIR="etsir" + echo "Linking sources..." + ln -srfT "$REPO/audiopicker/src/main/ets" $SRC + echo "Serializing..." + npx ts-node $SCRIPT -p $SRC $ETSIR -v +} + +prepare_audiopicker From 2e4737a5890de894d7ff5f51f27d11584c131384 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 19:12:41 +0300 Subject: [PATCH 037/120] Improve script for preparing projects, add EtsScene loader --- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 15 ++-- .../org/jacodb/ets/test/utils/LoadEts.kt | 12 +++ .../src/test/resources/prepare_projects.sh | 85 ++++++++++++------- 3 files changed, 72 insertions(+), 40 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 96bdb346e..262ed9e98 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -40,9 +40,8 @@ import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.dto.convertToEtsMethod import org.jacodb.ets.model.EtsClassSignature import org.jacodb.ets.model.EtsMethodSignature -import org.jacodb.ets.model.EtsScene import org.jacodb.ets.test.utils.loadEtsFileDtoFromResource -import org.jacodb.ets.test.utils.loadMultipleEtsFilesFromResourceDirectory +import org.jacodb.ets.test.utils.loadEtsProjectFromResources import org.jacodb.ets.utils.loadEtsFileAutoConvert import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test @@ -65,7 +64,7 @@ class EtsFromJsonTest { returnType = EtsAnyType, ) - private const val PROJECT_PATH = "/projects/AppSamples/ArkTSDistributedCalc" + private const val PROJECT_PATH = "/projects/ArkTSDistributedCalc" } @Test @@ -94,11 +93,11 @@ class EtsFromJsonTest { @EnabledIf("projectAvailable") @Test fun testLoadEtsProject() { - val path = "/projects/AppSamples/ArkTSDistributedCalc" - val files = loadMultipleEtsFilesFromResourceDirectory("$path/etsir").toList() - println("Loaded ${files.size} files") - val project = EtsScene(files) - println("project = $project") + val modules = listOf( + "entry", + ) + val prefix = "$PROJECT_PATH/etsir" + val project = loadEtsProjectFromResources(modules, prefix) println("Classes: ${project.classes.size}") for (cls in project.classes) { println("= ${cls.signature} with ${cls.methods.size} methods:") diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt index 0670d447a..e62bc8e87 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt @@ -20,6 +20,7 @@ import mu.KotlinLogging import org.jacodb.ets.dto.EtsFileDto import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.model.EtsFile +import org.jacodb.ets.model.EtsScene import java.nio.file.Path import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.extension @@ -80,6 +81,17 @@ fun loadMultipleEtsFilesFromMultipleResourceDirectories( return dirPaths.asSequence().flatMap { loadMultipleEtsFilesFromResourceDirectory(it) } } +fun loadEtsProjectFromResources( + modules: List, + prefix: String, +): EtsScene { + logger.info { "Loading Ets project with modules $modules from '$prefix/'" } + val dirPaths = modules.map { "$prefix/$it" } + val files = loadMultipleEtsFilesFromMultipleResourceDirectories(dirPaths).toList() + logger.info { "Loaded ${files.size} files" } + return EtsScene(files) +} + //----------------------------------------------------------------------------- /** diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index ef20b2128..e909a8554 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -14,65 +14,86 @@ mkdir -p projects pushd projects function check_repo() { + if [[ $# -ne 1 ]]; then + echo "Usage: check_repo " + exit 1 + fi if [[ ! -d $1 ]]; then echo "Repository not found: $1" exit 1 fi } -function prepare_app_samples() { - NAME="AppSamples" +function prepare_module() { + if [[ $# -ne 2 ]]; then + echo "Usage: prepare_module " + exit 1 + fi + local MODULE=$1 + local ROOT=$2 + echo "= Preparing module: $MODULE" + local SRC="source/$MODULE" + local ETSIR="etsir/$MODULE" + mkdir -p $(dirname $SRC) + echo "Linking sources..." + echo "pwd = $(pwd)" + ln -srfT "$ROOT/src/main/ets" $SRC + echo "Serializing..." + npx ts-node $SCRIPT -p $SRC $ETSIR -v +} + +( + NAME="ArkTSDistributedCalc" echo echo "=== Preparing $NAME..." echo if [ -d $NAME ]; then echo "Directory already exists: $NAME" - return + exit fi mkdir -p $NAME - pushd $NAME >/dev/null + pushd $NAME REPO="../../repos/applications_app_samples" check_repo $REPO - function prepare_calc() { - NAME="ArkTSDistributedCalc" - echo "= Preparing subproject: $NAME" - mkdir -p $NAME - SRC="$NAME/source" - ETSIR="$NAME/etsir" - echo "Linking sources..." - ln -srfT "$REPO/code/SuperFeature/DistributedAppDev/$NAME/entry/src/main/ets" $SRC - echo "Serializing..." - npx ts-node $SCRIPT -p $SRC $ETSIR -v - } - - prepare_calc -} - -prepare_app_samples + prepare_module "entry" "$REPO/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc/entry" +) -function prepare_audiopicker() { +( NAME="AudioPicker" echo echo "=== Preparing $NAME..." echo - if [ -d $NAME ]; then + if [[ -d $NAME ]]; then echo "Directory already exists: $NAME" - return + exit fi mkdir -p $NAME - pushd $NAME >/dev/null + pushd $NAME REPO="../../repos/applications_filepicker" check_repo $REPO - SRC="source" - ETSIR="etsir" - echo "Linking sources..." - ln -srfT "$REPO/audiopicker/src/main/ets" $SRC - echo "Serializing..." - npx ts-node $SCRIPT -p $SRC $ETSIR -v -} + prepare_module "entry" "$REPO/entry" + prepare_module "audiopicker" "$REPO/audiopicker" +) + +( + NAME="Launcher" + echo + echo "=== Preparing $NAME..." + echo + if [[ -d $NAME ]]; then + echo "Directory already exists: $NAME" + exit + fi + mkdir -p $NAME + pushd $NAME + + REPO="../../repos/applications_launcher" + check_repo $REPO -prepare_audiopicker + prepare_module "common" "$REPO/common" + prepare_module "phone_launcher" "$REPO/product/phone" +) From 7c6e6106a84be65d2402734960a5f0bd0063a795 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 19:17:59 +0300 Subject: [PATCH 038/120] Extract common code for project dir preparation --- .../src/test/resources/prepare_projects.sh | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index e909a8554..f7dff96f8 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -24,6 +24,23 @@ function check_repo() { fi } +function prepare_project_dir() { + if [[ $# -ne 1 ]]; then + echo "Usage: prepare_project " + exit 1 + fi + NAME=$1 + echo + echo "=== Preparing $NAME..." + echo + if [[ -d $NAME ]]; then + echo "Directory already exists: $NAME" + exit + fi + mkdir -p $NAME + pushd $NAME +} + function prepare_module() { if [[ $# -ne 2 ]]; then echo "Usage: prepare_module " @@ -43,16 +60,7 @@ function prepare_module() { } ( - NAME="ArkTSDistributedCalc" - echo - echo "=== Preparing $NAME..." - echo - if [ -d $NAME ]; then - echo "Directory already exists: $NAME" - exit - fi - mkdir -p $NAME - pushd $NAME + prepare_project_dir "ArkTSDistributedCalc" REPO="../../repos/applications_app_samples" check_repo $REPO @@ -61,16 +69,7 @@ function prepare_module() { ) ( - NAME="AudioPicker" - echo - echo "=== Preparing $NAME..." - echo - if [[ -d $NAME ]]; then - echo "Directory already exists: $NAME" - exit - fi - mkdir -p $NAME - pushd $NAME + prepare_project_dir "AudioPicker" REPO="../../repos/applications_filepicker" check_repo $REPO @@ -80,16 +79,7 @@ function prepare_module() { ) ( - NAME="Launcher" - echo - echo "=== Preparing $NAME..." - echo - if [[ -d $NAME ]]; then - echo "Directory already exists: $NAME" - exit - fi - mkdir -p $NAME - pushd $NAME + prepare_project_dir "Launcher" REPO="../../repos/applications_launcher" check_repo $REPO From b89d197ed39f39417481d864565787103c6e91b0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 19:54:35 +0300 Subject: [PATCH 039/120] Add getResourcePathOrNull --- .../src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt index 365a9f324..f70b994e3 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt @@ -20,10 +20,13 @@ import java.io.InputStream import java.nio.file.Path import kotlin.io.path.toPath -fun getResourcePath(res: String): Path { +fun getResourcePathOrNull(res: String): Path? { require(res.startsWith("/")) { "Resource path must start with '/': '$res'" } return object {}::class.java.getResource(res)?.toURI()?.toPath() - ?: error("Resource not found: '$res'") +} + +fun getResourcePath(res: String): Path { + return getResourcePathOrNull(res) ?: error("Resource not found: '$res'") } fun getResourceStream(res: String): InputStream { From bab94af2e74efe94aecdab27b71376d92946c8e3 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 20:00:08 +0300 Subject: [PATCH 040/120] Use dynamic tests for loading local projects --- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 262ed9e98..0195c0fcd 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -19,6 +19,7 @@ package org.jacodb.ets.test import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import mu.KotlinLogging import org.jacodb.ets.base.EtsAnyType import org.jacodb.ets.base.EtsInstLocation import org.jacodb.ets.base.EtsLocal @@ -40,15 +41,22 @@ import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.dto.convertToEtsMethod import org.jacodb.ets.model.EtsClassSignature import org.jacodb.ets.model.EtsMethodSignature +import org.jacodb.ets.test.utils.getResourcePath +import org.jacodb.ets.test.utils.getResourcePathOrNull import org.jacodb.ets.test.utils.loadEtsFileDtoFromResource import org.jacodb.ets.test.utils.loadEtsProjectFromResources import org.jacodb.ets.utils.loadEtsFileAutoConvert import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.DynamicTest import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestFactory import org.junit.jupiter.api.condition.EnabledIf +import kotlin.io.path.div import kotlin.io.path.exists import kotlin.io.path.toPath +private val logger = KotlinLogging.logger {} + class EtsFromJsonTest { companion object { @@ -107,6 +115,56 @@ class EtsFromJsonTest { } } + @TestFactory + fun testLoadAllAvailableEtsProjects(): Collection { + val p = getResourcePathOrNull("/projects") ?: run { + logger.warn { "No projects found" } + return emptyList() + } + val availableProjectNames = p.toFile().listFiles { f -> f.isDirectory }!!.map { it.name } + logger.info { + buildString { + appendLine("Found projects: ${availableProjectNames.size}") + for (name in availableProjectNames) { + appendLine(" - $name") + } + } + } + return availableProjectNames.map { projectName -> + DynamicTest.dynamicTest("load $projectName") { + dynamicLoadEtsProject(projectName) + } + } + } + + private fun dynamicLoadEtsProject(projectName: String) { + val projectPath = getResourcePath("/projects/$projectName") + logger.info { "Loading project: $projectName" } + val etsirPath = projectPath / "etsir" + if (!etsirPath.exists()) { + logger.warn { "No etsir directory found for project $projectName" } + return + } + val modules = etsirPath.toFile().listFiles { f -> f.isDirectory }!!.map { it.name } + logger.info { "Found ${modules.size} modules: $modules" } + if (modules.isEmpty()) { + logger.warn { "No modules found for project $projectName" } + return + } + val project = loadEtsProjectFromResources(modules, "/projects/$projectName/etsir") + logger.info { + buildString { + appendLine("Loaded project with ${project.classes.size} classes and ${project.classes.sumOf { it.methods.size }} methods") + for (cls in project.classes) { + appendLine("= ${cls.signature} with ${cls.methods.size} methods:") + for (method in cls.methods) { + appendLine(" - ${method.signature}") + } + } + } + } + } + @Test fun testLoadValueFromJson() { val jsonString = """ From 88af61d67e9dbcbe6cdf049468503c01b0598ca0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 25 Sep 2024 20:52:05 +0300 Subject: [PATCH 041/120] Add DSL for dynamic tests --- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 20 ++++-- .../jacodb/ets/test/utils/TestFactoryDsl.kt | 68 +++++++++++++++++++ 2 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 0195c0fcd..b9e6d9f16 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -45,9 +45,9 @@ import org.jacodb.ets.test.utils.getResourcePath import org.jacodb.ets.test.utils.getResourcePathOrNull import org.jacodb.ets.test.utils.loadEtsFileDtoFromResource import org.jacodb.ets.test.utils.loadEtsProjectFromResources +import org.jacodb.ets.test.utils.testFactory import org.jacodb.ets.utils.loadEtsFileAutoConvert import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.DynamicTest import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestFactory import org.junit.jupiter.api.condition.EnabledIf @@ -116,10 +116,10 @@ class EtsFromJsonTest { } @TestFactory - fun testLoadAllAvailableEtsProjects(): Collection { + fun testLoadAllAvailableEtsProjects() = testFactory { val p = getResourcePathOrNull("/projects") ?: run { - logger.warn { "No projects found" } - return emptyList() + logger.warn { "No projects directory found in resources" } + return@testFactory } val availableProjectNames = p.toFile().listFiles { f -> f.isDirectory }!!.map { it.name } logger.info { @@ -130,9 +130,15 @@ class EtsFromJsonTest { } } } - return availableProjectNames.map { projectName -> - DynamicTest.dynamicTest("load $projectName") { - dynamicLoadEtsProject(projectName) + if (availableProjectNames.isEmpty()) { + logger.warn { "No projects found" } + return@testFactory + } + container("load ${availableProjectNames.size} projects") { + for (projectName in availableProjectNames) { + test("load $projectName") { + dynamicLoadEtsProject(projectName) + } } } } diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt new file mode 100644 index 000000000..7c5250a73 --- /dev/null +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.ets.test.utils + +import org.junit.jupiter.api.DynamicContainer +import org.junit.jupiter.api.DynamicNode +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.function.Executable +import java.util.stream.Stream + +private interface TestProvider { + fun test(name: String, test: () -> Unit) +} + +private interface ContainerProvider { + fun container(name: String, init: TestContainerBuilder.() -> Unit) +} + +class TestContainerBuilder(var name: String) : TestProvider, ContainerProvider { + private val nodes: MutableList = mutableListOf() + + override fun test(name: String, test: () -> Unit) { + nodes += dynamicTest(name, test) + } + + override fun container(name: String, init: TestContainerBuilder.() -> Unit) { + nodes += containerBuilder(name, init) + } + + fun build(): DynamicContainer = DynamicContainer.dynamicContainer(name, nodes) +} + +private fun containerBuilder(name: String, init: TestContainerBuilder.() -> Unit): DynamicContainer = + TestContainerBuilder(name).apply(init).build() + +class TestFactoryBuilder : TestProvider, ContainerProvider { + private val nodes: MutableList = mutableListOf() + + override fun test(name: String, test: () -> Unit) { + nodes += dynamicTest(name, test) + } + + override fun container(name: String, init: TestContainerBuilder.() -> Unit) { + nodes += containerBuilder(name, init) + } + + fun build(): Stream = nodes.stream() +} + +fun testFactory(init: TestFactoryBuilder.() -> Unit): Stream = + TestFactoryBuilder().apply(init).build() + +private fun dynamicTest(name: String, test: () -> Unit): DynamicTest = + DynamicTest.dynamicTest(name, Executable(test)) From 5d82cdfdef2f37fb0b512d707b0fa101aad598e5 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 11:16:03 +0300 Subject: [PATCH 042/120] More modules and projects --- .../src/test/resources/prepare_projects.sh | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index f7dff96f8..ab86e3a39 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -84,6 +84,30 @@ function prepare_module() { REPO="../../repos/applications_launcher" check_repo $REPO - prepare_module "common" "$REPO/common" + prepare_module "launcher_common" "$REPO/common" + prepare_module "launcher_appcenter" "$REPO/feature/appcenter" + prepare_module "launcher_bigfolder" "$REPO/feature/bigfolder" + prepare_module "launcher_form" "$REPO/feature/form" + prepare_module "launcher_gesturenavigation" "$REPO/feature/gesturenavigation" + prepare_module "launcher_numbadge" "$REPO/feature/numbadge" + prepare_module "launcher_pagedesktop" "$REPO/feature/pagedesktop" + prepare_module "launcher_recents" "$REPO/feature/recents" + prepare_module "launcher_smartDock" "$REPO/feature/smartdock" prepare_module "phone_launcher" "$REPO/product/phone" + prepare_module "pad_launcher" "$REPO/product/pad" + prepare_module "launcher_settings" "$REPO/feature/settings" +) + +( + prepare_project_dir "Settings" + + REPO="../../repos/applications_settings" + check_repo $REPO + + prepare_module "phone" "$REPO/product/phone" + # prepare_module "wearable" "$REPO/product/wearable" + prepare_module "component" "$REPO/common/component" + prepare_module "search" "$REPO/common/search" + ### prepare_module "settingsBase" "$REPO/common/settingsBase" + prepare_module "utils" "$REPO/common/utils" ) From 53a17fa328d6d6e7db4ce3336e8564178f01a43b Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 12:28:44 +0300 Subject: [PATCH 043/120] Move test utils to fixtures --- jacodb-ets/build.gradle.kts | 3 +++ .../kotlin/org/jacodb/ets/test/utils/LoadEts.kt | 0 .../kotlin/org/jacodb/ets/test/utils/Resources.kt} | 0 .../kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt | 0 4 files changed, 3 insertions(+) rename jacodb-ets/src/{test => testFixtures}/kotlin/org/jacodb/ets/test/utils/LoadEts.kt (100%) rename jacodb-ets/src/{test/kotlin/org/jacodb/ets/test/utils/Utils.kt => testFixtures/kotlin/org/jacodb/ets/test/utils/Resources.kt} (100%) rename jacodb-ets/src/{test => testFixtures}/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt (100%) diff --git a/jacodb-ets/build.gradle.kts b/jacodb-ets/build.gradle.kts index 2d18f61d5..48ad2553c 100644 --- a/jacodb-ets/build.gradle.kts +++ b/jacodb-ets/build.gradle.kts @@ -2,6 +2,7 @@ import java.io.FileNotFoundException plugins { kotlin("plugin.serialization") + `java-test-fixtures` } dependencies { @@ -17,6 +18,8 @@ dependencies { testImplementation(project(":jacodb-analysis")) testImplementation(testFixtures(project(":jacodb-core"))) testImplementation(Libs.mockk) + + testFixturesImplementation(Libs.junit_jupiter_api) } // Example usage: diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt b/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/LoadEts.kt similarity index 100% rename from jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/LoadEts.kt rename to jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/LoadEts.kt diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt b/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/Resources.kt similarity index 100% rename from jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/Utils.kt rename to jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/Resources.kt diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt b/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt similarity index 100% rename from jacodb-ets/src/test/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt rename to jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/TestFactoryDsl.kt From 2bfcf62ac1317e60f051f2fdc6efc9ee0851be5d Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 13:02:41 +0300 Subject: [PATCH 044/120] Reorder logging --- .../src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index b9e6d9f16..0ded282b9 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -144,8 +144,8 @@ class EtsFromJsonTest { } private fun dynamicLoadEtsProject(projectName: String) { - val projectPath = getResourcePath("/projects/$projectName") logger.info { "Loading project: $projectName" } + val projectPath = getResourcePath("/projects/$projectName") val etsirPath = projectPath / "etsir" if (!etsirPath.exists()) { logger.warn { "No etsir directory found for project $projectName" } From 384fdba4be7f2cf60772540a87d5a64dfa4fe7ad Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 15:23:35 +0300 Subject: [PATCH 045/120] Use node --- .../src/test/resources/prepare_projects.sh | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index ab86e3a39..c7ef7b7d3 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -7,7 +7,22 @@ if [[ -z "${ARKANALYZER_DIR}" ]]; then fi echo "ARKANALYZER_DIR=${ARKANALYZER_DIR}" -SCRIPT=$ARKANALYZER_DIR/src/save/serializeArkIR.ts +SCRIPT_TS=$ARKANALYZER_DIR/src/save/serializeArkIR.ts +SCRIPT_JS=$ARKANALYZER_DIR/out/src/save/serializeArkIR.js + +if [[ ! -f $SCRIPT_JS ]]; then + echo "Script not found: $SCRIPT_JS" + echo "Did you forget to build the ArkAnalyzer project?" + echo "Run 'npm run build' in the ArkAnalyzer project directory" + exit 1 +fi + +if [[ $SCRIPT_JS -ot $SCRIPT_TS ]]; then + echo "Script is outdated: $SCRIPT_JS" + echo "Did you forget to re-build the ArkAnalyzer project?" + echo "Run 'npm run build' in the ArkAnalyzer project directory" + exit 1 +fi pushd "$(dirname $0)" >/dev/null mkdir -p projects @@ -56,7 +71,9 @@ function prepare_module() { echo "pwd = $(pwd)" ln -srfT "$ROOT/src/main/ets" $SRC echo "Serializing..." - npx ts-node $SCRIPT -p $SRC $ETSIR -v + # TODO: add switch for using npx/node + # npx ts-node $SCRIPT_TS -p $SRC $ETSIR -v + node $SCRIPT_JS -p $SRC $ETSIR -v } ( From 7356b460410a455be931c86b51565c16eaff7ff0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 15:23:45 +0300 Subject: [PATCH 046/120] Add force mode --- jacodb-ets/src/test/resources/prepare_projects.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index c7ef7b7d3..b5373ffea 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -50,7 +50,10 @@ function prepare_project_dir() { echo if [[ -d $NAME ]]; then echo "Directory already exists: $NAME" - exit + # If `-f` (force mode) is not provided, exit the preparation for this project: + if [[ "$1" != "-f" ]]; then + exit 1 + fi fi mkdir -p $NAME pushd $NAME From 165c05d07a54142a779b774d3a19009776c204a2 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 15:25:11 +0300 Subject: [PATCH 047/120] Fix exit code Remove exit code (default `exit 0`) in order to simply skip the project setup (without any error) in the case when the project directory already exists, instead of stopping the whole script with an error. --- jacodb-ets/src/test/resources/prepare_projects.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index b5373ffea..4b7685094 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -52,7 +52,7 @@ function prepare_project_dir() { echo "Directory already exists: $NAME" # If `-f` (force mode) is not provided, exit the preparation for this project: if [[ "$1" != "-f" ]]; then - exit 1 + exit fi fi mkdir -p $NAME From 7383f96ee813f7340a44aa21ad3b6b08feb79c65 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 15:53:29 +0300 Subject: [PATCH 048/120] Add force mode (`-f` flag) for overriding existing dirs --- jacodb-ets/src/test/resources/prepare_projects.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 4b7685094..002d2866e 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -24,6 +24,17 @@ if [[ $SCRIPT_JS -ot $SCRIPT_TS ]]; then exit 1 fi +do_force=0 +while getopts ":f" opt; do + case $opt in + f) do_force=1 + echo "Force mode enabled" + ;; + *) printf "Illegal option '-%s'\n" "$opt" && exit 1 + ;; + esac +done + pushd "$(dirname $0)" >/dev/null mkdir -p projects pushd projects @@ -51,7 +62,7 @@ function prepare_project_dir() { if [[ -d $NAME ]]; then echo "Directory already exists: $NAME" # If `-f` (force mode) is not provided, exit the preparation for this project: - if [[ "$1" != "-f" ]]; then + if [[ $do_force -eq 0 ]]; then exit fi fi From 0106e07551c866d4afc79def6449aff81e842184 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 15:54:12 +0300 Subject: [PATCH 049/120] Make ts-node FASTER --- jacodb-ets/src/test/resources/prepare_projects.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 002d2866e..1025fa60a 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -86,7 +86,7 @@ function prepare_module() { ln -srfT "$ROOT/src/main/ets" $SRC echo "Serializing..." # TODO: add switch for using npx/node - # npx ts-node $SCRIPT_TS -p $SRC $ETSIR -v + # npx ts-node --files --transpileOnly $SCRIPT_TS -p $SRC $ETSIR -v node $SCRIPT_JS -p $SRC $ETSIR -v } From 8810018ae000222a74f7fd25cb4958e9d25264ab Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 15:54:24 +0300 Subject: [PATCH 050/120] Comment the check for outdated JS script --- jacodb-ets/src/test/resources/prepare_projects.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 1025fa60a..f64b96d9f 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -17,12 +17,12 @@ if [[ ! -f $SCRIPT_JS ]]; then exit 1 fi -if [[ $SCRIPT_JS -ot $SCRIPT_TS ]]; then - echo "Script is outdated: $SCRIPT_JS" - echo "Did you forget to re-build the ArkAnalyzer project?" - echo "Run 'npm run build' in the ArkAnalyzer project directory" - exit 1 -fi +#if [[ $SCRIPT_JS -ot $SCRIPT_TS ]]; then +# echo "Script is outdated: $SCRIPT_JS" +# echo "Did you forget to re-build the ArkAnalyzer project?" +# echo "Run 'npm run build' in the ArkAnalyzer project directory" +# exit 1 +#fi do_force=0 while getopts ":f" opt; do From aded53401f29dd2d795fca369028b60cf227d6f6 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 16:02:55 +0300 Subject: [PATCH 051/120] Space --- jacodb-ets/src/test/resources/prepare_projects.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index f64b96d9f..5127d2781 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -25,6 +25,7 @@ fi #fi do_force=0 + while getopts ":f" opt; do case $opt in f) do_force=1 From 8759d4f042cef717dffc819edff5cd391db22d03 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 26 Sep 2024 16:05:05 +0300 Subject: [PATCH 052/120] Add SettingsData project --- jacodb-ets/src/test/resources/prepare_projects.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 5127d2781..dcaa6dbd5 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -143,3 +143,12 @@ function prepare_module() { ### prepare_module "settingsBase" "$REPO/common/settingsBase" prepare_module "utils" "$REPO/common/utils" ) + +( + prepare_project_dir "SettingsData" + + REPO="../../repos/applications_settings_data" + check_repo $REPO + + prepare_module "entry" "$REPO/entry" +) From b959cf2551721a6751144861ee396a143e04733b Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 11:25:09 +0300 Subject: [PATCH 053/120] Use cd instead of pushd --- jacodb-ets/src/test/resources/prepare_projects.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index dcaa6dbd5..91f23930b 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -36,9 +36,9 @@ while getopts ":f" opt; do esac done -pushd "$(dirname $0)" >/dev/null +cd "$(dirname $0)" mkdir -p projects -pushd projects +cd projects function check_repo() { if [[ $# -ne 1 ]]; then @@ -68,7 +68,7 @@ function prepare_project_dir() { fi fi mkdir -p $NAME - pushd $NAME + cd $NAME } function prepare_module() { From 8d1e880d552383db6536ce45d93ab126bfe042a5 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 11:25:30 +0300 Subject: [PATCH 054/120] Add script for cloning/updating all repos with projects --- .../src/test/resources/prepare_repos.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 jacodb-ets/src/test/resources/prepare_repos.sh diff --git a/jacodb-ets/src/test/resources/prepare_repos.sh b/jacodb-ets/src/test/resources/prepare_repos.sh new file mode 100644 index 000000000..bdefb3a0c --- /dev/null +++ b/jacodb-ets/src/test/resources/prepare_repos.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +cd "$(dirname $0)" +mkdir -p repos +cd repos + +function prepare_repo() { + if [[ $# -ne 1 ]]; then + echo "Usage: prepare_repo " + exit 1 + fi + REPO=$1 + DIR=$(basename -s .git $(git ls-remote --get-url $REPO)) + if [[ ! -d $DIR ]]; then + echo "Cloning repository: $REPO" + git clone $REPO + else + echo "Pulling latest changes: $REPO" + git -C $DIR pull + fi +} + +prepare_repo https://gitee.com/openharmony/applications_app_samples +prepare_repo https://gitee.com/openharmony/applications_filepicker +prepare_repo https://gitee.com/openharmony/applications_hap +prepare_repo https://gitee.com/openharmony/applications_launcher +prepare_repo https://gitee.com/openharmony/applications_settings +prepare_repo https://gitee.com/openharmony/applications_settings_data +prepare_repo https://gitee.com/openharmony/applications_systemui From 7bb77c6a836b3fe95e43a593b8dea1a479ac7ff9 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 11:36:36 +0300 Subject: [PATCH 055/120] Cleanup --- jacodb-ets/src/test/resources/prepare_projects.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 91f23930b..dff131222 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -58,7 +58,7 @@ function prepare_project_dir() { fi NAME=$1 echo - echo "=== Preparing $NAME..." + echo "=== Preparing project: $NAME" echo if [[ -d $NAME ]]; then echo "Directory already exists: $NAME" From 3dd92de9d97a204438403e6d9bb26a38dfec6cf8 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 11:38:43 +0300 Subject: [PATCH 056/120] Fix messages --- jacodb-ets/src/test/resources/prepare_repos.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_repos.sh b/jacodb-ets/src/test/resources/prepare_repos.sh index bdefb3a0c..18d547cb0 100644 --- a/jacodb-ets/src/test/resources/prepare_repos.sh +++ b/jacodb-ets/src/test/resources/prepare_repos.sh @@ -12,11 +12,12 @@ function prepare_repo() { fi REPO=$1 DIR=$(basename -s .git $(git ls-remote --get-url $REPO)) + echo "Preparing repository: $REPO" if [[ ! -d $DIR ]]; then - echo "Cloning repository: $REPO" + echo "Cloning..." git clone $REPO else - echo "Pulling latest changes: $REPO" + echo "Directory '$DIR' already exists. Pulling latest changes..." git -C $DIR pull fi } From eba04add679acf37c484075ed2c5536921927aa4 Mon Sep 17 00:00:00 2001 From: Alexey Menshutin Date: Fri, 27 Sep 2024 16:04:33 +0300 Subject: [PATCH 057/120] Type inference hacks and improvements --- .../org/jacodb/ets/utils/LoadEtsFile.kt | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index cb6a057cf..7e385fe13 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -19,14 +19,18 @@ package org.jacodb.ets.utils import org.jacodb.ets.dto.EtsFileDto import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.model.EtsFile +import org.jacodb.ets.model.EtsScene import java.io.FileNotFoundException import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.Path import kotlin.io.path.absolute +import kotlin.io.path.createTempDirectory import kotlin.io.path.exists import kotlin.io.path.inputStream import kotlin.io.path.nameWithoutExtension import kotlin.io.path.pathString +import kotlin.io.path.walk import kotlin.time.Duration.Companion.seconds private const val ENV_VAR_ARK_ANALYZER_DIR = "ARKANALYZER_DIR" @@ -38,14 +42,14 @@ private const val DEFAULT_SERIALIZE_SCRIPT_PATH = "out/src/save/serializeArkIR.j private const val ENV_VAR_NODE_EXECUTABLE = "NODE_EXECUTABLE" private const val DEFAULT_NODE_EXECUTABLE = "node" -fun generateEtsFileIR(tsPath: Path): Path { +fun generateEtsIR(path: Path, isProject: Boolean = false): Path { val arkAnalyzerDir = Path(System.getenv(ENV_VAR_ARK_ANALYZER_DIR) ?: DEFAULT_ARK_ANALYZER_DIR) if (!arkAnalyzerDir.exists()) { throw FileNotFoundException( "ArkAnalyzer directory does not exist: '${arkAnalyzerDir.absolute()}'. " + - "Did you forget to set the '$ENV_VAR_ARK_ANALYZER_DIR' environment variable? " + - "Current value is '${System.getenv(ENV_VAR_ARK_ANALYZER_DIR)}', " + - "current dir is '${Path("").toAbsolutePath()}'." + "Did you forget to set the '$ENV_VAR_ARK_ANALYZER_DIR' environment variable? " + + "Current value is '${System.getenv(ENV_VAR_ARK_ANALYZER_DIR)}', " + + "current dir is '${Path("").toAbsolutePath()}'." ) } @@ -54,27 +58,49 @@ fun generateEtsFileIR(tsPath: Path): Path { if (!script.exists()) { throw FileNotFoundException( "Script file not found: '$script'. " + - "Did you forget to execute 'npm run build' in the arkanalyzer project?" + "Did you forget to execute 'npm run build' in the arkanalyzer project?" ) } val node = System.getenv(ENV_VAR_NODE_EXECUTABLE) ?: DEFAULT_NODE_EXECUTABLE - val output = kotlin.io.path.createTempFile(prefix = tsPath.nameWithoutExtension, suffix = ".json") - val cmd: List = listOf( + val output = if (isProject) { + createTempDirectory(prefix = path.nameWithoutExtension) + } else { + kotlin.io.path.createTempFile(prefix = path.nameWithoutExtension, suffix = ".json") + } + + val cmd = listOfNotNull( node, script.pathString, - tsPath.pathString, + if (isProject) "-p" else null, + path.pathString, output.pathString, ) - runProcess(cmd, 60.seconds) + runProcess(cmd, 10.seconds) return output } -fun loadEtsFileAutoConvert(tsPath: Path): EtsFile { - val irFilePath = generateEtsFileIR(tsPath) +fun loadEtsFileAutoConvert(path: Path): EtsFile { + val irFilePath = generateEtsIR(path, isProject = false) + irFilePath.inputStream().use { stream -> val etsFileDto = EtsFileDto.loadFromJson(stream) val etsFile = convertToEtsFile(etsFileDto) return etsFile } } + +@OptIn(ExperimentalPathApi::class) +fun loadEtsProjectAutoConvert(path: Path): EtsScene { + val irFolderPath = generateEtsIR(path, isProject = true) + + val files = irFolderPath + .walk() + .filter { it.toFile().extension == "json" } + .mapTo(mutableListOf()) { + val etsFileDto = EtsFileDto.loadFromJson(it.inputStream()) + convertToEtsFile(etsFileDto) + } + + return EtsScene(files) +} From 484f7bd47edab5bcf1fbd70596a5f32ac119a6b2 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 17:05:22 +0300 Subject: [PATCH 058/120] Cleanup --- .../org/jacodb/ets/utils/LoadEtsFile.kt | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index 7e385fe13..93df4b21c 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -27,6 +27,7 @@ import kotlin.io.path.Path import kotlin.io.path.absolute import kotlin.io.path.createTempDirectory import kotlin.io.path.exists +import kotlin.io.path.extension import kotlin.io.path.inputStream import kotlin.io.path.nameWithoutExtension import kotlin.io.path.pathString @@ -47,9 +48,9 @@ fun generateEtsIR(path: Path, isProject: Boolean = false): Path { if (!arkAnalyzerDir.exists()) { throw FileNotFoundException( "ArkAnalyzer directory does not exist: '${arkAnalyzerDir.absolute()}'. " + - "Did you forget to set the '$ENV_VAR_ARK_ANALYZER_DIR' environment variable? " + - "Current value is '${System.getenv(ENV_VAR_ARK_ANALYZER_DIR)}', " + - "current dir is '${Path("").toAbsolutePath()}'." + "Did you forget to set the '$ENV_VAR_ARK_ANALYZER_DIR' environment variable? " + + "Current value is '${System.getenv(ENV_VAR_ARK_ANALYZER_DIR)}', " + + "current dir is '${Path("").toAbsolutePath()}'." ) } @@ -58,7 +59,7 @@ fun generateEtsIR(path: Path, isProject: Boolean = false): Path { if (!script.exists()) { throw FileNotFoundException( "Script file not found: '$script'. " + - "Did you forget to execute 'npm run build' in the arkanalyzer project?" + "Did you forget to execute 'npm run build' in the arkanalyzer project?" ) } @@ -85,22 +86,22 @@ fun loadEtsFileAutoConvert(path: Path): EtsFile { irFilePath.inputStream().use { stream -> val etsFileDto = EtsFileDto.loadFromJson(stream) - val etsFile = convertToEtsFile(etsFileDto) - return etsFile + return convertToEtsFile(etsFileDto) } } @OptIn(ExperimentalPathApi::class) fun loadEtsProjectAutoConvert(path: Path): EtsScene { val irFolderPath = generateEtsIR(path, isProject = true) - val files = irFolderPath .walk() - .filter { it.toFile().extension == "json" } - .mapTo(mutableListOf()) { - val etsFileDto = EtsFileDto.loadFromJson(it.inputStream()) - convertToEtsFile(etsFileDto) + .filter { it.extension == "json" } + .map { + it.inputStream().use { stream -> + val etsFileDto = EtsFileDto.loadFromJson(stream) + convertToEtsFile(etsFileDto) + } } - + .toList() return EtsScene(files) } From 2277a3ef4d85400d869911b3c887f992dd9dbb98 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 18:23:25 +0300 Subject: [PATCH 059/120] Move Maybe to jacodb-core --- .../main/kotlin/org/jacodb/analysis/config/Condition.kt | 4 ++-- .../src/main/kotlin/org/jacodb/analysis/config/Position.kt | 7 +++---- .../main/kotlin/org/jacodb/analysis/config/TaintAction.kt | 6 +++--- .../org/jacodb/analysis/impl/ConditionEvaluatorTest.kt | 4 ++-- .../src/main/kotlin/org/jacodb/impl/util}/Maybe.kt | 4 +++- 5 files changed, 13 insertions(+), 12 deletions(-) rename {jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds => jacodb-core/src/main/kotlin/org/jacodb/impl/util}/Maybe.kt (95%) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt index af05a9dd1..376fae3db 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt @@ -16,8 +16,6 @@ package org.jacodb.analysis.config -import org.jacodb.analysis.ifds.Maybe -import org.jacodb.analysis.ifds.onSome import org.jacodb.analysis.taint.Tainted import org.jacodb.analysis.util.Traits import org.jacodb.analysis.util.removeTrailingElementAccessors @@ -26,6 +24,8 @@ import org.jacodb.api.common.cfg.CommonInst import org.jacodb.api.common.cfg.CommonValue import org.jacodb.api.jvm.cfg.JcValue import org.jacodb.api.jvm.ext.isAssignable +import org.jacodb.impl.util.Maybe +import org.jacodb.impl.util.onSome import org.jacodb.taint.configuration.And import org.jacodb.taint.configuration.AnnotationType import org.jacodb.taint.configuration.ConditionVisitor diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt index e0842c397..347e72497 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt @@ -18,16 +18,15 @@ package org.jacodb.analysis.config import org.jacodb.analysis.ifds.AccessPath import org.jacodb.analysis.ifds.ElementAccessor -import org.jacodb.analysis.ifds.Maybe -import org.jacodb.analysis.ifds.fmap -import org.jacodb.analysis.ifds.toMaybe import org.jacodb.analysis.util.Traits import org.jacodb.api.common.CommonMethod -import org.jacodb.api.common.CommonProject import org.jacodb.api.common.cfg.CommonAssignInst import org.jacodb.api.common.cfg.CommonInst import org.jacodb.api.common.cfg.CommonInstanceCallExpr import org.jacodb.api.common.cfg.CommonValue +import org.jacodb.impl.util.Maybe +import org.jacodb.impl.util.fmap +import org.jacodb.impl.util.toMaybe import org.jacodb.taint.configuration.AnyArgument import org.jacodb.taint.configuration.Argument import org.jacodb.taint.configuration.Position diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/TaintAction.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/TaintAction.kt index 59403b099..b35a48326 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/TaintAction.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/TaintAction.kt @@ -17,10 +17,10 @@ package org.jacodb.analysis.config import org.jacodb.analysis.ifds.AccessPath -import org.jacodb.analysis.ifds.Maybe -import org.jacodb.analysis.ifds.fmap -import org.jacodb.analysis.ifds.map import org.jacodb.analysis.taint.Tainted +import org.jacodb.impl.util.Maybe +import org.jacodb.impl.util.fmap +import org.jacodb.impl.util.map import org.jacodb.taint.configuration.AssignMark import org.jacodb.taint.configuration.CopyAllMarks import org.jacodb.taint.configuration.CopyMark diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt index c99bb98e8..f31050cf5 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt @@ -20,8 +20,6 @@ import io.mockk.every import io.mockk.mockk import org.jacodb.analysis.config.BasicConditionEvaluator import org.jacodb.analysis.config.FactAwareConditionEvaluator -import org.jacodb.analysis.ifds.Maybe -import org.jacodb.analysis.ifds.toMaybe import org.jacodb.analysis.taint.Tainted import org.jacodb.analysis.util.JcTraits import org.jacodb.api.jvm.JcClasspath @@ -34,6 +32,8 @@ import org.jacodb.api.jvm.cfg.JcInt import org.jacodb.api.jvm.cfg.JcStringConstant import org.jacodb.api.jvm.cfg.JcThis import org.jacodb.api.jvm.cfg.JcValue +import org.jacodb.impl.util.Maybe +import org.jacodb.impl.util.toMaybe import org.jacodb.taint.configuration.And import org.jacodb.taint.configuration.AnnotationType import org.jacodb.taint.configuration.Argument diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Maybe.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/util/Maybe.kt similarity index 95% rename from jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Maybe.kt rename to jacodb-core/src/main/kotlin/org/jacodb/impl/util/Maybe.kt index 0ccec0670..808d2c590 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Maybe.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/util/Maybe.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.jacodb.analysis.ifds +package org.jacodb.impl.util @JvmInline value class Maybe private constructor( @@ -65,3 +65,5 @@ inline fun Maybe.onNone(body: () -> Unit): Maybe { } fun T?.toMaybe(): Maybe = Maybe.from(this) + +fun Maybe.getOrNull(): T? = if (isSome) getOrThrow() else null From a85cbd059d1bbec9f78e39d72328c88b97350b35 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 18:24:29 +0300 Subject: [PATCH 060/120] Speed up Ets IFDS by 5917% --- jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsCfg.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsCfg.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsCfg.kt index e7fda91b0..48de74a06 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsCfg.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsCfg.kt @@ -49,8 +49,8 @@ class EtsCfg( override val entries: List get() = listOfNotNull(stmts.firstOrNull()) - override val exits: List - get() = instructions.filterIsInstance() + override val exits: List = + instructions.filterIsInstance() override fun successors(node: EtsStmt): Set { return successorMap[node]!!.toSet() From baab36bb4a810064a90646c06a5424ca17a6de22 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 18:24:54 +0300 Subject: [PATCH 061/120] New callees resolver --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 152 +++++++++++++----- 1 file changed, 116 insertions(+), 36 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 21fadf158..0ff652274 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -16,13 +16,19 @@ package org.jacodb.ets.graph +import mu.KotlinLogging import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.ets.base.EtsStmt +import org.jacodb.ets.model.EtsClass import org.jacodb.ets.model.EtsClassSignature import org.jacodb.ets.model.EtsFileSignature import org.jacodb.ets.model.EtsMethod +import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.model.EtsScene import org.jacodb.ets.utils.callExpr +import org.jacodb.impl.util.Maybe + +private val logger = KotlinLogging.logger {} interface EtsApplicationGraph : ApplicationGraph { val cp: EtsScene @@ -34,6 +40,9 @@ private fun EtsFileSignature?.isUnknown(): Boolean = private fun EtsClassSignature.isUnknown(): Boolean = name.isBlank() +private fun EtsClassSignature.isIdeal(): Boolean = + !isUnknown() && !file.isUnknown() + enum class ComparisonResult { Equal, NotEqual, @@ -76,66 +85,137 @@ class EtsApplicationGraphImpl( return successors.asSequence() } + private val cacheClassWithIdealSignature: MutableMap> = hashMapOf() + private val cacheMethodWithIdealSignature: MutableMap> = hashMapOf() + private val cachePartiallyMatchedCallees: MutableMap> = hashMapOf() + + private fun lookupClassWithIdealSignature(signature: EtsClassSignature): Maybe { + require(signature.isIdeal()) + + if (signature in cacheClassWithIdealSignature) { + return cacheClassWithIdealSignature.getValue(signature) + } + + val matched = cp.classes + .asSequence() + .filter { it.signature == signature && it.signature.file == signature.file } + if (matched.none()) { + cacheClassWithIdealSignature[signature] = Maybe.none() + return Maybe.none() + } else { + val s = matched.singleOrNull() + ?: error("Multiple classes with the same signature: ${matched.toList()}") + cacheClassWithIdealSignature[signature] = Maybe.some(s) + return Maybe.some(s) + } + } + override fun callees(node: EtsStmt): Sequence { val expr = node.callExpr ?: return emptySequence() val callee = expr.method // Note: the resolving code below expects that at least the current method signature is known. - check(!node.method.enclosingClass.isUnknown()) { - "Unknown class signature in method: ${node.method}" + check(node.method.enclosingClass.isIdeal()) { + "Incomplete signature in method: ${node.method}" } // Note: specific resolve for constructor: if (callee.name == "constructor") { - val resolvedCompletely = cp.classes - .asSequence() - .filter { it.signature == callee.enclosingClass } - .map { it.ctor } - if (resolvedCompletely.any()) return resolvedCompletely - - val resolvedPartially = cp.classes - .asSequence() - .filter { it.signature.name == callee.enclosingClass.name } - .map { it.ctor } - .toList() - if (resolvedPartially.size == 1) return resolvedPartially.asSequence() - return emptySequence() + if (!callee.enclosingClass.isIdeal()) { + // Constructor signature is garbage. Sorry, can't do anything in such case. + return emptySequence() + } + + // Here, we assume that the constructor signature is ideal. + check(callee.enclosingClass.isIdeal()) + + val cls = lookupClassWithIdealSignature(callee.enclosingClass) + if (cls.isSome) { + return sequenceOf(cls.getOrThrow().ctor) + } else { + return emptySequence() + } } - // First, try to resolve the callee via a complete signature match: - val resolvedCompletely = cp.classes - .asSequence() - .filter { compareClassSignatures(it.signature, callee.enclosingClass) == ComparisonResult.Equal } - // Note: include constructors! - .flatMap { it.methods.asSequence() + it.ctor } - .filter { it.name == callee.name } - if (resolvedCompletely.any()) return resolvedCompletely + // If the callee signature is ideal, resolve it directly: + if (callee.enclosingClass.isIdeal()) { + if (callee in cacheMethodWithIdealSignature) { + val resolved = cacheMethodWithIdealSignature.getValue(callee) + if (resolved.isSome) { + return sequenceOf(resolved.getOrThrow()) + } else { + return emptySequence() + } + } + + val cls = lookupClassWithIdealSignature(callee.enclosingClass) + + val resolved = run { + if (cls.isNone) { + emptySequence() + } else { + cls.getOrThrow().methods.asSequence().filter { it.name == callee.name } + } + } + if (resolved.none()) { + cacheMethodWithIdealSignature[callee] = Maybe.none() + return emptySequence() + } + val r = resolved.singleOrNull() + ?: error("Multiple methods with the same signature: ${resolved.toList()}") + cacheMethodWithIdealSignature[callee] = Maybe.some(r) + return sequenceOf(r) + } + + // If the callee signature is not ideal, resolve it via a partial match... + check(!callee.enclosingClass.isIdeal()) + + val cls = lookupClassWithIdealSignature(node.method.enclosingClass).let { + if (it.isNone) { + error("Could not find the enclosing class: ${node.method.enclosingClass}") + } + it.getOrThrow() + } // If the complete signature match failed, - // try to find the unique neighbour (non-recursive) method in the same class: - val resolvedNeighbour = cp.classes - .single { it.signature == node.method.enclosingClass } - .methods + // try to find the unique not-the-same neighbour method in the same class: + val neighbors = cls.methods .asSequence() .filter { it.name == callee.name } .filterNot { it.name == node.method.name } - if (resolvedNeighbour.any()) return resolvedNeighbour + if (neighbors.any()) { + val s = neighbors.singleOrNull() + ?: error("Multiple methods with the same name: ${neighbors.toList()}") + cachePartiallyMatchedCallees[callee] = listOf(s) + return sequenceOf(s) + } + + // NOTE: cache lookup MUST be performed AFTER trying to match the neighbour! + if (callee in cachePartiallyMatchedCallees) { + return cachePartiallyMatchedCallees.getValue(callee).asSequence() + } // If the neighbour match failed, // try to *uniquely* resolve the callee via a partial signature match: - val resolvedPartially = cp.classes + val resolved = cp.classes .asSequence() .filter { compareClassSignatures(it.signature, callee.enclosingClass) != ComparisonResult.NotEqual } + // Note: exclude current class: + .filterNot { compareClassSignatures(it.signature, node.method.enclosingClass) != ComparisonResult.NotEqual } // Note: omit constructors! .flatMap { it.methods.asSequence() } .filter { it.name == callee.name } - // Note: exclude recursive calls: - .filterNot { it.name == node.method.name } - - if (resolvedPartially.none()) return emptySequence() - val resolved = resolvedPartially.toList() - if (resolved.size == 1) return resolved.asSequence() - return emptySequence() + if (resolved.none()) { + cachePartiallyMatchedCallees[callee] = emptyList() + return emptySequence() + } + val r = resolved.singleOrNull() ?: run { + logger.warn { "Multiple methods with the same signature: ${resolved.toList()}" } + cachePartiallyMatchedCallees[callee] = emptyList() + return emptySequence() + } + cachePartiallyMatchedCallees[callee] = listOf(r) + return sequenceOf(r) } override fun callers(method: EtsMethod): Sequence { From c23966845b08b84d79e9abd59d91e57e1a6a17f3 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 27 Sep 2024 18:30:27 +0300 Subject: [PATCH 062/120] Fix imports --- .../src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt | 2 +- .../main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt index 84251e6ba..42f573f49 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt @@ -30,7 +30,6 @@ import org.jacodb.analysis.ifds.FlowFunctions import org.jacodb.analysis.ifds.isOnHeap import org.jacodb.analysis.ifds.isStatic import org.jacodb.analysis.ifds.minus -import org.jacodb.analysis.ifds.onSome import org.jacodb.analysis.taint.TaintDomainFact import org.jacodb.analysis.taint.TaintZeroFact import org.jacodb.analysis.taint.Tainted @@ -60,6 +59,7 @@ import org.jacodb.api.jvm.cfg.JcNullConstant import org.jacodb.api.jvm.cfg.JcReturnInst import org.jacodb.api.jvm.ext.findType import org.jacodb.api.jvm.ext.isNullable +import org.jacodb.impl.util.onSome import org.jacodb.taint.configuration.AssignMark import org.jacodb.taint.configuration.CopyAllMarks import org.jacodb.taint.configuration.CopyMark diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt index f028b72be..6a4cf1abf 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt @@ -29,7 +29,6 @@ import org.jacodb.analysis.ifds.FlowFunctions import org.jacodb.analysis.ifds.isOnHeap import org.jacodb.analysis.ifds.isStatic import org.jacodb.analysis.ifds.minus -import org.jacodb.analysis.ifds.onSome import org.jacodb.analysis.util.Traits import org.jacodb.analysis.util.startsWith import org.jacodb.api.common.CommonMethod @@ -52,6 +51,7 @@ import org.jacodb.ets.base.EtsBinaryExpr import org.jacodb.ets.base.EtsCastExpr import org.jacodb.ets.base.EtsUnaryExpr import org.jacodb.ets.utils.getOperands +import org.jacodb.impl.util.onSome import org.jacodb.taint.configuration.AssignMark import org.jacodb.taint.configuration.CopyAllMarks import org.jacodb.taint.configuration.CopyMark From 5a970bc6e929b5b1bd17787d3c4c68b85612ec7e Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 30 Sep 2024 13:18:15 +0300 Subject: [PATCH 063/120] Cleanup dot-view extension for JcGraph --- .../kotlin/org/jacodb/impl/cfg/GraphExt.kt | 39 +++++++++++++++---- .../org/jacodb/testing/cfg/IRSvgGenerator.kt | 4 +- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt index 3fcd50b45..14954b6a3 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt @@ -58,20 +58,32 @@ import java.io.File import java.nio.file.Files import java.nio.file.Path -fun JcGraph.view(dotCmd: String, viewerCmd: String, viewCatchConnections: Boolean = false) { - Util.sh(arrayOf(viewerCmd, "file://${toFile(dotCmd, viewCatchConnections)}")) +private const val DEFAULT_DOT_CMD = "dot" + +fun JcGraph.view( + viewerCmd: String = if (System.getProperty("os.name").lowercase().contains("windows")) "start" else "xdg-open", + dotCmd: String = DEFAULT_DOT_CMD, + viewCatchConnections: Boolean = false, +) { + val path = toFile(null, dotCmd, viewCatchConnections) + Util.sh(arrayOf(viewerCmd, "file://$path")) } -fun JcGraph.toFile(dotCmd: String, viewCatchConnections: Boolean = false, file: File? = null): Path { +fun JcGraph.toFile( + file: File? = null, + dotCmd: String = DEFAULT_DOT_CMD, + viewCatchConnections: Boolean = false, +): Path { Graph.setDefaultCmd(dotCmd) val graph = Graph("jcGraph") val nodes = mutableMapOf() for ((index, inst) in instructions.withIndex()) { + val label = inst.toString().replace("\"", "\\\"") val node = Node("$index") .setShape(Shape.box) - .setLabel(inst.toString().replace("\"", "\\\"")) + .setLabel(label) .setFontSize(12.0) nodes[inst] = node graph.addNode(node) @@ -140,20 +152,31 @@ fun JcGraph.toFile(dotCmd: String, viewCatchConnections: Boolean = false, file: return resultingFile } -fun JcBlockGraph.view(dotCmd: String, viewerCmd: String) { - Util.sh(arrayOf(viewerCmd, "file://${toFile(dotCmd)}")) +fun JcBlockGraph.view( + viewerCmd: String, + dotCmd: String = DEFAULT_DOT_CMD, +) { + val path = toFile(null, dotCmd = dotCmd) + Util.sh(arrayOf(viewerCmd, "file://$path")) } -fun JcBlockGraph.toFile(dotCmd: String, file: File? = null): Path { +fun JcBlockGraph.toFile( + file: File? = null, + dotCmd: String = DEFAULT_DOT_CMD, +): Path { Graph.setDefaultCmd(dotCmd) val graph = Graph("jcGraph") val nodes = mutableMapOf() for ((index, block) in instructions.withIndex()) { + val label = instructions(block) + .joinToString("") { "$it\\l" } + .replace("\"", "\\\"") + .replace("\n", "\\n") val node = Node("$index") .setShape(Shape.box) - .setLabel(instructions(block).joinToString("") { "$it\\l" }.replace("\"", "\\\"").replace("\n", "\\n")) + .setLabel(label) .setFontSize(12.0) nodes[block] = node graph.addNode(node) diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRSvgGenerator.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRSvgGenerator.kt index 61604e857..efb0bb14f 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRSvgGenerator.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRSvgGenerator.kt @@ -56,8 +56,8 @@ class IRSvgGenerator(private val folder: File) : Closeable { val fileName = "${it.enclosingClass.simpleName}-$fixedName-$index.svg" val graph = it.flowGraph() JcGraphChecker(it, graph).check() - graph.toFile("dot", false, file = File(folder, "graph-$fileName")) - graph.blockGraph().toFile("dot", file = File(folder, "block-graph-$fileName")) + graph.toFile(File(folder, "graph-$fileName")) + graph.blockGraph().toFile(File(folder, "block-graph-$fileName")) } } From dcd037e645ad643c4eea2ea9f2f14196fecb6e59 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 30 Sep 2024 13:19:50 +0300 Subject: [PATCH 064/120] Cleanup dot-view extension for EtsCfg --- .../kotlin/org/jacodb/ets/utils/EtsCfgExt.kt | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt index 671b8204c..fbedc1f66 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt @@ -29,20 +29,32 @@ import java.io.File import java.nio.file.Files import java.nio.file.Path -fun EtsCfg.view(dotCmd: String, viewerCmd: String, viewCatchConnections: Boolean = false) { - Util.sh(arrayOf(viewerCmd, "file://${toFile(dotCmd, viewCatchConnections)}")) +private const val DEFAULT_DOT_CMD = "dot" + +fun EtsCfg.view( + viewerCmd: String = if (System.getProperty("os.name").startsWith("Windows")) "start" else "xdg-open", + dotCmd: String = DEFAULT_DOT_CMD, + viewCatchConnections: Boolean = false, +) { + val path = toFile(null, dotCmd, viewCatchConnections) + Util.sh(arrayOf(viewerCmd, "file://$path")) } -fun EtsCfg.toFile(dotCmd: String, viewCatchConnections: Boolean = false, file: File? = null): Path { +fun EtsCfg.toFile( + file: File? = null, + dotCmd: String = DEFAULT_DOT_CMD, + viewCatchConnections: Boolean = false, +): Path { Graph.setDefaultCmd(dotCmd) val graph = Graph("etsCfg") val nodes = mutableMapOf() for ((index, inst) in instructions.withIndex()) { + val label = inst.toString().replace("\"", "\\\"") val node = Node("$index") .setShape(Shape.box) - .setLabel(inst.toString().replace("\"", "\\\"")) + .setLabel(label) .setFontSize(12.0) nodes[inst] = node graph.addNode(node) From 67a2f9a375bed871418fffac4d210240347ed2d4 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 30 Sep 2024 13:20:29 +0300 Subject: [PATCH 065/120] Fix messages --- .../main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 0ff652274..66a24cda1 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -162,7 +162,7 @@ class EtsApplicationGraphImpl( return emptySequence() } val r = resolved.singleOrNull() - ?: error("Multiple methods with the same signature: ${resolved.toList()}") + ?: error("Multiple methods with the same complete signature: ${resolved.toList()}") cacheMethodWithIdealSignature[callee] = Maybe.some(r) return sequenceOf(r) } @@ -210,7 +210,7 @@ class EtsApplicationGraphImpl( return emptySequence() } val r = resolved.singleOrNull() ?: run { - logger.warn { "Multiple methods with the same signature: ${resolved.toList()}" } + logger.warn { "Multiple methods with the same partial signature '${callee}': ${resolved.toList()}" } cachePartiallyMatchedCallees[callee] = emptyList() return emptySequence() } From a81887b80feda3772a5453ab85c315f864c47937 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 30 Sep 2024 13:21:41 +0300 Subject: [PATCH 066/120] Change default for viewCatchConnections to true in EtsCfg.view --- jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt index fbedc1f66..76a7332af 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt @@ -34,7 +34,7 @@ private const val DEFAULT_DOT_CMD = "dot" fun EtsCfg.view( viewerCmd: String = if (System.getProperty("os.name").startsWith("Windows")) "start" else "xdg-open", dotCmd: String = DEFAULT_DOT_CMD, - viewCatchConnections: Boolean = false, + viewCatchConnections: Boolean = true, ) { val path = toFile(null, dotCmd, viewCatchConnections) Util.sh(arrayOf(viewerCmd, "file://$path")) @@ -43,7 +43,7 @@ fun EtsCfg.view( fun EtsCfg.toFile( file: File? = null, dotCmd: String = DEFAULT_DOT_CMD, - viewCatchConnections: Boolean = false, + viewCatchConnections: Boolean = true, ): Path { Graph.setDefaultCmd(dotCmd) From 99b274154ad3cd2ecde6747417e8572ef020de4f Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 30 Sep 2024 15:26:14 +0300 Subject: [PATCH 067/120] Move graph settings --- .../src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt index 76a7332af..ec4af0508 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt @@ -48,6 +48,9 @@ fun EtsCfg.toFile( Graph.setDefaultCmd(dotCmd) val graph = Graph("etsCfg") + .setBgColor(Color.X11.transparent) + .setFontSize(12.0) + .setFontName("Fira Mono") val nodes = mutableMapOf() for ((index, inst) in instructions.withIndex()) { @@ -60,10 +63,6 @@ fun EtsCfg.toFile( graph.addNode(node) } - graph.setBgColor(Color.X11.transparent) - graph.setFontSize(12.0) - graph.setFontName("Fira Mono") - for ((inst, node) in nodes) { when (inst) { is EtsIfStmt -> { From 4f4af9a65d5eb102f93544032eca2cff82dd2eff Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 30 Sep 2024 19:40:48 +0300 Subject: [PATCH 068/120] Support "incorrect" IfStmt without 2 successors in dot-view --- .../kotlin/org/jacodb/ets/utils/EtsCfgExt.kt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt index ec4af0508..d04db630c 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsCfgExt.kt @@ -67,19 +67,22 @@ fun EtsCfg.toFile( when (inst) { is EtsIfStmt -> { val successors = successors(inst).toList() - check(successors.size == 2) + // check(successors.size == 2) + check(successors.size <= 2) graph.addEdge( Edge(node.name, nodes[successors[0]]!!.name) .also { it.setLabel("false") } ) - graph.addEdge( - Edge(node.name, nodes[successors[1]]!!.name) - .also { - it.setLabel("true") - } - ) + if (successors.size == 2) { + graph.addEdge( + Edge(node.name, nodes[successors[1]]!!.name) + .also { + it.setLabel("true") + } + ) + } } else -> for (successor in successors(inst)) { From 2b777c0e6b46b32a8441ad3042405c9ea4bc7f62 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 2 Oct 2024 15:21:32 +0300 Subject: [PATCH 069/120] Fix ArrayTypeDto::toString --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index abfd9beae..461ea9c42 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -179,7 +179,7 @@ data class ArrayTypeDto( val dimensions: Int, ) : TypeDto { override fun toString(): String { - return "$elementType[]".repeat(dimensions) + return "$elementType" + "[]".repeat(dimensions) } } From 00f4dfce57da0b011c7290c605efb8069c66177c Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 2 Oct 2024 15:22:14 +0300 Subject: [PATCH 070/120] Add GenericTypeDto --- .../src/main/kotlin/org/jacodb/ets/dto/Types.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index 461ea9c42..463046427 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -193,6 +193,18 @@ data class UnclearReferenceTypeDto( } } +@Serializable +@SerialName("GenericType") +data class GenericTypeDto( + val name: String, + val defaultType: TypeDto? = null, + val constraint: TypeDto? = null, +) : TypeDto { + override fun toString(): String { + return name + (constraint?.let { " extends $it" } ?: "") + (defaultType?.let { " = $it" } ?: "") + } +} + @Serializable @SerialName("UNKNOWN_TYPE") data class AbsolutelyUnknownTypeDto( From 460b628ec1b014496920ae07831a962ac75df0c6 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 2 Oct 2024 15:33:16 +0300 Subject: [PATCH 071/120] Add EtsGenericType --- .../kotlin/org/jacodb/ets/base/EtsType.kt | 19 ++++++++++++++ .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 25 +++++++++++++------ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt index b5e567c47..520df37c6 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt @@ -45,6 +45,7 @@ interface EtsType : CommonType, CommonTypeName { fun visit(type: EtsArrayType): R fun visit(type: EtsArrayObjectType): R fun visit(type: EtsUnclearRefType): R + fun visit(type: EtsGenericType): R interface Default : Visitor { override fun visit(type: EtsAnyType): R = defaultVisit(type) @@ -64,6 +65,7 @@ interface EtsType : CommonType, CommonTypeName { override fun visit(type: EtsArrayType): R = defaultVisit(type) override fun visit(type: EtsArrayObjectType): R = defaultVisit(type) override fun visit(type: EtsUnclearRefType): R = defaultVisit(type) + override fun visit(type: EtsGenericType): R = defaultVisit(type) fun defaultVisit(type: EtsType): R } @@ -276,3 +278,20 @@ data class EtsUnclearRefType( return visitor.visit(this) } } + +data class EtsGenericType( + val name: String, + val defaultType: EtsType? = null, + val constraint: EtsType? = null, +) : EtsRefType { + override val typeName: String + get() = name + + override fun toString(): String { + return name + (constraint?.let { " extends $it" } ?: "") + (defaultType?.let { " = $it" } ?: "") + } + + override fun accept(visitor: EtsType.Visitor): R { + return visitor.visit(this) + } +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 1e8860fd6..36233b745 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -44,6 +44,7 @@ import org.jacodb.ets.base.EtsExpExpr import org.jacodb.ets.base.EtsExpr import org.jacodb.ets.base.EtsFieldRef import org.jacodb.ets.base.EtsFunctionType +import org.jacodb.ets.base.EtsGenericType import org.jacodb.ets.base.EtsGotoStmt import org.jacodb.ets.base.EtsGtEqExpr import org.jacodb.ets.base.EtsGtExpr @@ -606,30 +607,38 @@ fun convertToEtsType(type: TypeDto): EtsType { dimensions = type.dimensions, ) - is FunctionTypeDto -> EtsFunctionType( - method = convertToEtsMethodSignature(type.signature) - ) + BooleanTypeDto -> EtsBooleanType is ClassTypeDto -> EtsClassType( classSignature = convertToEtsClassSignature(type.signature) ) - NeverTypeDto -> EtsNeverType + is FunctionTypeDto -> EtsFunctionType( + method = convertToEtsMethodSignature(type.signature) + ) - BooleanTypeDto -> EtsBooleanType + is GenericTypeDto -> { + val defaultType = type.defaultType?.let { convertToEtsType(it) } + val constraint = type.constraint?.let { convertToEtsType(it) } + EtsGenericType( + name = type.name, + defaultType = defaultType, + constraint = constraint, + ) + } is LiteralTypeDto -> EtsLiteralType( literalTypeName = type.literal.toString() ) + NeverTypeDto -> EtsNeverType + NullTypeDto -> EtsNullType NumberTypeDto -> EtsNumberType StringTypeDto -> EtsStringType - UndefinedTypeDto -> EtsUndefinedType - is TupleTypeDto -> EtsTupleType( types = type.types.map { convertToEtsType(it) } ) @@ -638,6 +647,8 @@ fun convertToEtsType(type: TypeDto): EtsType { typeName = type.name ) + UndefinedTypeDto -> EtsUndefinedType + is UnionTypeDto -> EtsUnionType( types = type.types.map { convertToEtsType(it) } ) From 2fa0a76b637ec6ffb175fa5717f75c508e6e6428 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 2 Oct 2024 16:27:24 +0300 Subject: [PATCH 072/120] Handle locals --- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 29 ++++++++++--------- .../kotlin/org/jacodb/ets/model/EtsMethod.kt | 6 ++-- .../main/kotlin/org/jacodb/ets/utils/Utils.kt | 3 +- .../kotlin/org/jacodb/ets/test/EtsFileTest.kt | 2 +- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 2 +- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 36233b745..bdea2b544 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -127,11 +127,10 @@ import org.jacodb.ets.model.EtsNamespaceSignature class EtsMethodBuilder( signature: EtsMethodSignature, - // Default locals count is args + this - localsCount: Int = signature.parameters.size + 1, + locals: List = emptyList(), modifiers: List = emptyList(), ) { - val etsMethod = EtsMethodImpl(signature, localsCount, modifiers) + val etsMethod = EtsMethodImpl(signature, locals, modifiers = modifiers) private val currentStmts: MutableList = mutableListOf() @@ -294,10 +293,7 @@ class EtsMethodBuilder( } } - is LocalDto -> EtsLocal( - name = value.name, - type = convertToEtsType(value.type), - ) + is LocalDto -> convertToEtsLocal(value) is ConstantDto -> convertToEtsConstant(value) @@ -750,15 +746,13 @@ fun convertToEtsMethod(method: MethodDto): EtsMethod { .filterIsInstance() .map { it.modifier } if (method.body != null) { - // Note: locals are not used in the current implementation - // val locals = method.body.locals.map { - // convertToEtsEntity(it) as EtsLocal // safe cast - // } - val localsCount = method.body.locals.size - val builder = EtsMethodBuilder(signature, localsCount, modifiers) + val locals = method.body.locals.map { + convertToEtsLocal(it) + } + val builder = EtsMethodBuilder(signature, locals, modifiers) return builder.build(method.body.cfg) } else { - return EtsMethodImpl(signature, modifiers = modifiers) + return EtsMethodImpl(signature, locals = emptyList(), modifiers = modifiers) } } @@ -801,3 +795,10 @@ fun convertToEtsFile(file: EtsFileDto): EtsFile { namespaces = namespaces, ) } + +fun convertToEtsLocal(local: LocalDto): EtsLocal { + return EtsLocal( + name = local.name, + type = convertToEtsType(local.type), + ) +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt index 44880a64f..47fb7de74 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt @@ -17,6 +17,7 @@ package org.jacodb.ets.model import org.jacodb.api.common.CommonMethod +import org.jacodb.ets.base.EtsLocal import org.jacodb.ets.base.EtsType import org.jacodb.ets.graph.EtsCfg @@ -24,8 +25,8 @@ import org.jacodb.ets.graph.EtsCfg // TODO: typeParameters interface EtsMethod : CommonMethod { val signature: EtsMethodSignature - val localsCount: Int val cfg: EtsCfg + val locals: List val modifiers: List val enclosingClass: EtsClassSignature @@ -60,8 +61,7 @@ interface EtsMethod : CommonMethod { class EtsMethodImpl( override val signature: EtsMethodSignature, - // Default locals count is args + this - override val localsCount: Int = signature.parameters.size + 1, + override val locals: List = emptyList(), override val modifiers: List = emptyList(), ) : EtsMethod { internal var _cfg: EtsCfg? = null diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt index e313c33fd..14e525564 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt @@ -112,6 +112,7 @@ fun EtsFile.toText(): String { lines += " - FIELD '${field.signature}'" } lines += " constructor = '${clazz.ctor.signature}'" + lines += " locals: ${clazz.ctor.locals.size}" lines += " stmts: ${clazz.ctor.cfg.stmts.size}" clazz.ctor.cfg.stmts.forEach { stmt -> lines += " ${stmt.location.index}. $stmt" @@ -121,7 +122,7 @@ fun EtsFile.toText(): String { lines += " methods: ${clazz.methods.size}" clazz.methods.forEach { method -> lines += " - METHOD '${method.signature}':" - lines += " locals = ${method.localsCount}" + lines += " locals: ${method.locals.size}" lines += " stmts: ${method.cfg.stmts.size}" method.cfg.stmts.forEach { stmt -> lines += " ${stmt.location.index}. $stmt" diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt index 09bbd6bf6..d339009f7 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt @@ -47,7 +47,7 @@ class EtsFileTest { etsFile.classes.forEach { cls -> cls.methods.forEach { method -> logger.info { - "Method '$method', locals: ${method.localsCount}, instructions: ${method.cfg.instructions.size}" + "Method '$method', locals: ${method.locals.size}, instructions: ${method.cfg.instructions.size}" } method.cfg.instructions.forEach { inst -> logger.info { "${inst.location.index}. $inst" } diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 0ded282b9..a301cd60a 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -280,7 +280,7 @@ class EtsFromJsonTest { ), method.signature ) - Assertions.assertEquals(0, method.localsCount) + Assertions.assertEquals(0, method.locals.size) Assertions.assertEquals(1, method.cfg.stmts.size) Assertions.assertEquals( listOf( From 56052a3892a19bce18f3f3cbd65b17053c1fb3df Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 2 Oct 2024 19:16:16 +0300 Subject: [PATCH 073/120] Update ArkAnalyzer branch --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f56a4fe44..604fca93a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-09-23" + BRANCH="neo/2024-10-02" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break From 90e81fd8f829a63a9abd01af738934bdcd21d321 Mon Sep 17 00:00:00 2001 From: Alexey Menshutin Date: Fri, 4 Oct 2024 13:30:07 +0300 Subject: [PATCH 074/120] Enable arkanalyzer's type inference --- .../main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index 93df4b21c..3518f77e1 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -43,7 +43,11 @@ private const val DEFAULT_SERIALIZE_SCRIPT_PATH = "out/src/save/serializeArkIR.j private const val ENV_VAR_NODE_EXECUTABLE = "NODE_EXECUTABLE" private const val DEFAULT_NODE_EXECUTABLE = "node" -fun generateEtsIR(path: Path, isProject: Boolean = false): Path { +fun generateEtsIR( + path: Path, + isProject: Boolean = false, + useArkAnalyzerTypeInference: Int? = null +): Path { val arkAnalyzerDir = Path(System.getenv(ENV_VAR_ARK_ANALYZER_DIR) ?: DEFAULT_ARK_ANALYZER_DIR) if (!arkAnalyzerDir.exists()) { throw FileNotFoundException( @@ -74,6 +78,7 @@ fun generateEtsIR(path: Path, isProject: Boolean = false): Path { node, script.pathString, if (isProject) "-p" else null, + useArkAnalyzerTypeInference?.let { "-t $it" }, path.pathString, output.pathString, ) @@ -82,7 +87,7 @@ fun generateEtsIR(path: Path, isProject: Boolean = false): Path { } fun loadEtsFileAutoConvert(path: Path): EtsFile { - val irFilePath = generateEtsIR(path, isProject = false) + val irFilePath = generateEtsIR(path, isProject = false, useArkAnalyzerTypeInference = 2) irFilePath.inputStream().use { stream -> val etsFileDto = EtsFileDto.loadFromJson(stream) @@ -92,7 +97,7 @@ fun loadEtsFileAutoConvert(path: Path): EtsFile { @OptIn(ExperimentalPathApi::class) fun loadEtsProjectAutoConvert(path: Path): EtsScene { - val irFolderPath = generateEtsIR(path, isProject = true) + val irFolderPath = generateEtsIR(path, isProject = true, useArkAnalyzerTypeInference = 2) val files = irFolderPath .walk() .filter { it.extension == "json" } From 54484ad59ab82d218c125345c59c961a10b88a0e Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 9 Oct 2024 14:11:31 +0300 Subject: [PATCH 075/120] Add more open-source projects --- .../src/test/resources/prepare_projects.sh | 226 +++++++++++++++++- .../src/test/resources/prepare_repos.sh | 11 +- 2 files changed, 233 insertions(+), 4 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index dff131222..b618dd009 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -92,16 +92,144 @@ function prepare_module() { } ( - prepare_project_dir "ArkTSDistributedCalc" + prepare_project_dir "Demo_Calc" REPO="../../repos/applications_app_samples" check_repo $REPO - prepare_module "entry" "$REPO/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc/entry" + BASE="$REPO/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc" + prepare_module "entry" "$BASE/entry" ) +( + prepare_project_dir "Demo_Camera" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/BasicFeature/Media/Camera" + prepare_module "entry" "$BASE/entry" +) +( + prepare_project_dir "Demo_CertificateManager" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/BasicFeature/Security/CertManager" + prepare_module "entry" "$BASE/entry" +) +( + prepare_project_dir "Demo_Clock" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/Solutions/Tools/ArkTSClock" + prepare_module "entry" "$BASE/entry" +) +( + prepare_project_dir "Demo_KikaInput" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + BASE="$REPO/code/Solutions/InputMethod/KikaInput" + prepare_module "entry" "$BASE/entry" +) ( - prepare_project_dir "AudioPicker" + prepare_project_dir "Demo_Launcher" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/SystemFeature/ApplicationModels/Launcher" + prepare_module "entry" "$BASE/entry" + prepare_module "desktop" "$BASE/desktop" + prepare_module "base" "$BASE/base" + prepare_module "recents" "$BASE/recents" +) +( + prepare_project_dir "Demo_Music" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/SuperFeature/DistributedAppDev/ArkTSDistributedMusicPlayer" + prepare_module "entry" "$BASE/entry" +) +( + prepare_project_dir "Demo_Photos" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/SystemFeature/FileManagement/Photos" + prepare_module "entry" "$BASE/entry" +) +( + prepare_project_dir "Demo_ScreenShot" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/SystemFeature/Media/Screenshot" + prepare_module "entry" "$BASE/entry" + prepare_module "Feature" "$BASE/Feature" +) +( + prepare_project_dir "Demo_Settings" + + REPO="../../repos/applications_app_samples" + check_repo $REPO + + BASE="$REPO/code/SuperFeature/MultiDeviceAppDev/Settings" + prepare_module "default" "$BASE/products/default" + prepare_module "common" "$BASE/common" + prepare_module "settingItems" "$BASE/features/settingitems" +) + +( + prepare_project_dir "CalendarData" + + REPO="../../repos/applications_calendar_data" + check_repo $REPO + + prepare_module "entry" "$REPO/entry" + prepare_module "common" "$REPO/common" + prepare_module "datastructure" "$REPO/datastructure" + prepare_module "datamanager" "$REPO/datamanager" + prepare_module "rrule" "$REPO/rrule" + prepare_module "dataprovider" "$REPO/dataprovider" +) + +( + prepare_project_dir "CallUI" + + REPO="../../repos/applications_call" + check_repo $REPO + + prepare_module "callui" "$REPO/entry" + prepare_module "common" "$REPO/common" + prepare_module "mobiledatasettings" "$REPO/mobiledatasettings" +) + +( + prepare_project_dir "Contacts" + + REPO="../../repos/applications_contacts" + check_repo $REPO + + prepare_module "entry" "$REPO/entry" + prepare_module "common" "$REPO/common" + prepare_module "phonenumber" "$REPO/feature/phonenumber" + prepare_module "contact" "$REPO/feature/contact" + prepare_module "account" "$REPO/feature/account" + prepare_module "call" "$REPO/feature/call" + prepare_module "dialpad" "$REPO/feature/dialpad" +) + +( + prepare_project_dir "FilePicker" REPO="../../repos/applications_filepicker" check_repo $REPO @@ -130,6 +258,59 @@ function prepare_module() { prepare_module "launcher_settings" "$REPO/feature/settings" ) +( + prepare_project_dir "Mms" + + REPO="../../repos/applications_mms" + check_repo $REPO + + prepare_module "entry" "$REPO/entry" +) + +( + prepare_project_dir "Note" + + REPO="../../repos/applications_notes" + check_repo $REPO + + prepare_module "default" "$REPO/product/default" + prepare_module "utils" "$REPO/common/utils" + # prepare_module "resources" "$REPO/common/resources" + prepare_module "component" "$REPO/features" +) + +( + prepare_project_dir "PrintSpooler" + + REPO="../../repos/applications_print_spooler" + check_repo $REPO + + prepare_module "entry" "$REPO/entry" + prepare_module "common" "$REPO/common" + prepare_module "ippPrint" "$REPO/feature/ippPrint" +) + +( + prepare_project_dir "ScreenLock" + + REPO="../../repos/applications_screenlock" + check_repo $REPO + + prepare_module "entry" "$REPO/entry" + prepare_module "pc" "$REPO/product/pc" + prepare_module "phone" "$REPO/product/phone" + prepare_module "batterycomponent" "$REPO/features/batterycomponent" + prepare_module "clockcomponent" "$REPO/features/clockcomponent" + prepare_module "datetimecomponent" "$REPO/features/datetimecomponent" + prepare_module "noticeitem" "$REPO/features/noticeitem" + prepare_module "screenlock" "$REPO/features/screenlock" + prepare_module "shortcutcomponent" "$REPO/features/shortcutcomponent" + prepare_module "signalcomponent" "$REPO/features/signalcomponent" + prepare_module "wallpapercomponent" "$REPO/features/wallpapercomponent" + prepare_module "wificomponent" "$REPO/features/wificomponent" + prepare_module "common" "$REPO/common" +) + ( prepare_project_dir "Settings" @@ -152,3 +333,42 @@ function prepare_module() { prepare_module "entry" "$REPO/entry" ) + +( + prepare_project_dir "SystemUI" + + REPO="../../repos/applications_systemui" + check_repo $REPO + + prepare_module "phone_entry" "$REPO/entry/phone" + prepare_module "pc_entry" "$REPO/entry/pc" + prepare_module "default_navigationBar" "$REPO/product/default/navigationBar" + prepare_module "default_notificationmanagement" "$REPO/product/default/notificationmanagement" + prepare_module "default_volumepanel" "$REPO/product/default/volumepanel" + prepare_module "default_dialog" "$REPO/product/default/dialog" + prepare_module "pc_controlpanel" "$REPO/product/pc/controlpanel" + prepare_module "pc_notificationpanel" "$REPO/product/pc/notificationpanel" + prepare_module "pc_statusbar" "$REPO/product/pc/statusbar" + prepare_module "phone_dropdownpanel" "$REPO/product/phone/dropdownpanel" + prepare_module "phone_statusbar" "$REPO/product/phone/statusbar" + prepare_module "common" "$REPO/common" + prepare_module "airplanecomponent" "$REPO/features/airplanecomponent" + prepare_module "autorotatecomponent" "$REPO/features/autorotatecomponent" + prepare_module "batterycomponent" "$REPO/features/batterycomponent" + prepare_module "bluetoothcomponent" "$REPO/features/bluetoothcomponent" + prepare_module "brightnesscomponent" "$REPO/features/brightnesscomponent" + prepare_module "capsulecomponent" "$REPO/features/capsulecomponent" + prepare_module "clockcomponent" "$REPO/features/clockcomponent" + prepare_module "controlcentercomponent" "$REPO/features/controlcentercomponent" + prepare_module "locationcomponent" "$REPO/features/locationcomponent" + prepare_module "managementcomponent" "$REPO/features/managementcomponent" + prepare_module "navigationservice" "$REPO/features/navigationservice" + prepare_module "nfccomponent" "$REPO/features/nfccomponent" + prepare_module "noticeitem" "$REPO/features/noticeitem" + prepare_module "ringmodecomponent" "$REPO/features/ringmodecomponent" + prepare_module "signalcomponent" "$REPO/features/signalcomponent" + prepare_module "statusbarcomponent" "$REPO/features/statusbarcomponent" + prepare_module "volumecomponent" "$REPO/features/volumecomponent" + prepare_module "volumepanelcomponent" "$REPO/features/volumepanelcomponent" + prepare_module "wificomponent" "$REPO/features/wificomponent" +) diff --git a/jacodb-ets/src/test/resources/prepare_repos.sh b/jacodb-ets/src/test/resources/prepare_repos.sh index 18d547cb0..5a684c3b3 100644 --- a/jacodb-ets/src/test/resources/prepare_repos.sh +++ b/jacodb-ets/src/test/resources/prepare_repos.sh @@ -18,14 +18,23 @@ function prepare_repo() { git clone $REPO else echo "Directory '$DIR' already exists. Pulling latest changes..." - git -C $DIR pull + git -C $DIR pull & fi } prepare_repo https://gitee.com/openharmony/applications_app_samples +prepare_repo https://gitee.com/openharmony/applications_calendar_data +prepare_repo https://gitee.com/openharmony/applications_call +prepare_repo https://gitee.com/openharmony/applications_contacts prepare_repo https://gitee.com/openharmony/applications_filepicker prepare_repo https://gitee.com/openharmony/applications_hap prepare_repo https://gitee.com/openharmony/applications_launcher +prepare_repo https://gitee.com/openharmony/applications_mms +prepare_repo https://gitee.com/openharmony/applications_notes +prepare_repo https://gitee.com/openharmony/applications_print_spooler +prepare_repo https://gitee.com/openharmony/applications_screenlock prepare_repo https://gitee.com/openharmony/applications_settings prepare_repo https://gitee.com/openharmony/applications_settings_data prepare_repo https://gitee.com/openharmony/applications_systemui + +wait From 48af899c2e620ba014ea5bca447758c2d6073642 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 9 Oct 2024 16:13:45 +0300 Subject: [PATCH 076/120] Convert to lists to avoid re-iterating sequences --- .../org/jacodb/ets/graph/EtsApplicationGraph.kt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 66a24cda1..1f7cd18bf 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -99,12 +99,13 @@ class EtsApplicationGraphImpl( val matched = cp.classes .asSequence() .filter { it.signature == signature && it.signature.file == signature.file } - if (matched.none()) { + .toList() + if (matched.isEmpty()) { cacheClassWithIdealSignature[signature] = Maybe.none() return Maybe.none() } else { val s = matched.singleOrNull() - ?: error("Multiple classes with the same signature: ${matched.toList()}") + ?: error("Multiple classes with the same signature: $matched") cacheClassWithIdealSignature[signature] = Maybe.some(s) return Maybe.some(s) } @@ -183,9 +184,10 @@ class EtsApplicationGraphImpl( .asSequence() .filter { it.name == callee.name } .filterNot { it.name == node.method.name } - if (neighbors.any()) { + .toList() + if (neighbors.isNotEmpty()) { val s = neighbors.singleOrNull() - ?: error("Multiple methods with the same name: ${neighbors.toList()}") + ?: error("Multiple methods with the same name: $neighbors") cachePartiallyMatchedCallees[callee] = listOf(s) return sequenceOf(s) } @@ -205,12 +207,13 @@ class EtsApplicationGraphImpl( // Note: omit constructors! .flatMap { it.methods.asSequence() } .filter { it.name == callee.name } - if (resolved.none()) { + .toList() + if (resolved.isEmpty()) { cachePartiallyMatchedCallees[callee] = emptyList() return emptySequence() } val r = resolved.singleOrNull() ?: run { - logger.warn { "Multiple methods with the same partial signature '${callee}': ${resolved.toList()}" } + logger.warn { "Multiple methods with the same partial signature '${callee}': $resolved" } cachePartiallyMatchedCallees[callee] = emptyList() return emptySequence() } From 94fd103d7ea385bd610fe9843d2444891673f0f6 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 11 Oct 2024 15:12:07 +0300 Subject: [PATCH 077/120] Add PrimitiveLiteralDto --- .../kotlin/org/jacodb/ets/dto/Serializers.kt | 39 ++++++++++++++++++- .../main/kotlin/org/jacodb/ets/dto/Types.kt | 18 ++++++++- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 20 ++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt index 81f73da5b..88a20dadb 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt @@ -14,6 +14,8 @@ * limitations under the License. */ +@file:OptIn(ExperimentalSerializationApi::class) + package org.jacodb.ets.dto import kotlinx.serialization.ExperimentalSerializationApi @@ -21,6 +23,8 @@ import kotlinx.serialization.InternalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.SerializationException import kotlinx.serialization.descriptors.PolymorphicKind +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.descriptors.buildSerialDescriptor import kotlinx.serialization.descriptors.element @@ -30,11 +34,14 @@ import kotlinx.serialization.json.JsonDecoder import kotlinx.serialization.json.JsonEncoder import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.json.double import kotlinx.serialization.json.encodeToJsonElement +import kotlinx.serialization.json.intOrNull object ModifierSerializer : KSerializer { - @OptIn(ExperimentalSerializationApi::class, InternalSerializationApi::class) + @OptIn(InternalSerializationApi::class) override val descriptor: SerialDescriptor = buildSerialDescriptor("Modifier", PolymorphicKind.SEALED) { element("DecoratorItem") @@ -59,3 +66,33 @@ object ModifierSerializer : KSerializer { } } } + +object PrimitiveLiteralSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("PrimitiveLiteral", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: PrimitiveLiteralDto) { + require(encoder is JsonEncoder) + when (value) { + is PrimitiveLiteralDto.StringLiteral -> encoder.encodeString(value.value) + is PrimitiveLiteralDto.NumberLiteral -> encoder.encodeDouble(value.value.toDouble()) + is PrimitiveLiteralDto.BooleanLiteral -> encoder.encodeBoolean(value.value) + } + } + + override fun deserialize(decoder: Decoder): PrimitiveLiteralDto { + require(decoder is JsonDecoder) + val element = decoder.decodeJsonElement() + if (element !is JsonPrimitive) { + throw SerializationException("Expected JsonPrimitive, but found $element") + } + if (element.isString) { + return PrimitiveLiteralDto.StringLiteral(element.content) + } + val b = element.booleanOrNull + if (b != null) { + return PrimitiveLiteralDto.BooleanLiteral(b) + } else { + return PrimitiveLiteralDto.NumberLiteral(element.double) + } + } +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index 463046427..e110cb42f 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -20,7 +20,6 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator -import kotlinx.serialization.json.JsonElement @Serializable @OptIn(ExperimentalSerializationApi::class) @@ -142,7 +141,7 @@ object UndefinedTypeDto : PrimitiveTypeDto { @Serializable @SerialName("LiteralType") data class LiteralTypeDto( - val literal: JsonElement, // string | boolean | number + val literal: PrimitiveLiteralDto, ) : PrimitiveTypeDto { override val name: String get() = literal.toString() @@ -152,6 +151,21 @@ data class LiteralTypeDto( } } +@Serializable(with = PrimitiveLiteralSerializer::class) +sealed class PrimitiveLiteralDto { + data class StringLiteral(val value: String) : PrimitiveLiteralDto() { + override fun toString(): String = value + } + + data class NumberLiteral(val value: Double) : PrimitiveLiteralDto() { + override fun toString(): String = value.toString() + } + + data class BooleanLiteral(val value: Boolean) : PrimitiveLiteralDto() { + override fun toString(): String = value.toString() + } +} + @Serializable @SerialName("ClassType") data class ClassTypeDto( diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index a301cd60a..f7a7771f6 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -31,12 +31,15 @@ import org.jacodb.ets.dto.EtsMethodBuilder import org.jacodb.ets.dto.FieldDto import org.jacodb.ets.dto.FieldSignatureDto import org.jacodb.ets.dto.FileSignatureDto +import org.jacodb.ets.dto.LiteralTypeDto import org.jacodb.ets.dto.LocalDto import org.jacodb.ets.dto.MethodDto import org.jacodb.ets.dto.ModifierDto import org.jacodb.ets.dto.NumberTypeDto +import org.jacodb.ets.dto.PrimitiveLiteralDto import org.jacodb.ets.dto.ReturnVoidStmtDto import org.jacodb.ets.dto.StmtDto +import org.jacodb.ets.dto.TypeDto import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.dto.convertToEtsMethod import org.jacodb.ets.model.EtsClassSignature @@ -54,6 +57,8 @@ import org.junit.jupiter.api.condition.EnabledIf import kotlin.io.path.div import kotlin.io.path.exists import kotlin.io.path.toPath +import kotlin.test.assertEquals +import kotlin.test.assertIs private val logger = KotlinLogging.logger {} @@ -328,4 +333,19 @@ class EtsFromJsonTest { modifiers ) } + + @Test + fun testLoadLiteralTypeFromJson() { + // TS: `let x: "hello" = "hello";` + val jsonString = """ + { + "_": "LiteralType", + "literal": "hello" + } + """.trimIndent() + val typeDto = Json.decodeFromString(jsonString) + println("typeDto = $typeDto") + assertIs(typeDto) + assertEquals(PrimitiveLiteralDto.StringLiteral("hello"), typeDto.literal) + } } From 2af482e4a1c88c9f738a79791e4630af86e7154c Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 11 Oct 2024 15:58:58 +0300 Subject: [PATCH 078/120] Run ArkAnalyzer's type inference on projects --- jacodb-ets/src/test/resources/prepare_projects.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index b618dd009..6e3dad460 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -88,7 +88,7 @@ function prepare_module() { echo "Serializing..." # TODO: add switch for using npx/node # npx ts-node --files --transpileOnly $SCRIPT_TS -p $SRC $ETSIR -v - node $SCRIPT_JS -p $SRC $ETSIR -v + node $SCRIPT_JS -p $SRC $ETSIR -v -t 2 } ( From 44bdca254bab6b0abff6198939205998ab85c91a Mon Sep 17 00:00:00 2001 From: Alexey Menshutin Date: Fri, 25 Oct 2024 12:57:46 +0300 Subject: [PATCH 079/120] First entry points support --- jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index 3518f77e1..af06bc7ca 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -78,6 +78,7 @@ fun generateEtsIR( node, script.pathString, if (isProject) "-p" else null, + "-e", // load entrypoints useArkAnalyzerTypeInference?.let { "-t $it" }, path.pathString, output.pathString, From 3f6f73f2800a273e15ccea4e49ba08e08182d5b6 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 28 Oct 2024 13:50:02 +0300 Subject: [PATCH 080/120] Update AA branch in CI --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 604fca93a..af903b005 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-10-02" + BRANCH="neo/2024-10-28" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break From 30ca455b31a941684c02a3452330a40f251fdca7 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 28 Oct 2024 18:09:14 +0300 Subject: [PATCH 081/120] Add `loadEntrypoint` argument for EtsIR loader --- .../org/jacodb/ets/utils/LoadEtsFile.kt | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index af06bc7ca..3180863d5 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -46,7 +46,8 @@ private const val DEFAULT_NODE_EXECUTABLE = "node" fun generateEtsIR( path: Path, isProject: Boolean = false, - useArkAnalyzerTypeInference: Int? = null + loadEntrypoints: Boolean = true, + useArkAnalyzerTypeInference: Int? = null, ): Path { val arkAnalyzerDir = Path(System.getenv(ENV_VAR_ARK_ANALYZER_DIR) ?: DEFAULT_ARK_ANALYZER_DIR) if (!arkAnalyzerDir.exists()) { @@ -78,7 +79,7 @@ fun generateEtsIR( node, script.pathString, if (isProject) "-p" else null, - "-e", // load entrypoints + if (loadEntrypoints) "-e" else null, useArkAnalyzerTypeInference?.let { "-t $it" }, path.pathString, output.pathString, @@ -88,8 +89,11 @@ fun generateEtsIR( } fun loadEtsFileAutoConvert(path: Path): EtsFile { - val irFilePath = generateEtsIR(path, isProject = false, useArkAnalyzerTypeInference = 2) - + val irFilePath = generateEtsIR( + path, + isProject = false, + useArkAnalyzerTypeInference = 2 + ) irFilePath.inputStream().use { stream -> val etsFileDto = EtsFileDto.loadFromJson(stream) return convertToEtsFile(etsFileDto) @@ -97,8 +101,16 @@ fun loadEtsFileAutoConvert(path: Path): EtsFile { } @OptIn(ExperimentalPathApi::class) -fun loadEtsProjectAutoConvert(path: Path): EtsScene { - val irFolderPath = generateEtsIR(path, isProject = true, useArkAnalyzerTypeInference = 2) +fun loadEtsProjectAutoConvert( + path: Path, + loadEntrypoints: Boolean = false, +): EtsScene { + val irFolderPath = generateEtsIR( + path, + isProject = true, + loadEntrypoints = loadEntrypoints, + useArkAnalyzerTypeInference = 2, + ) val files = irFolderPath .walk() .filter { it.extension == "json" } From 6bad23dbdbcd753931ae64623a55f4abdadd4a8f Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 28 Oct 2024 18:09:25 +0300 Subject: [PATCH 082/120] Use getResourcePath --- .../src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index f7a7771f6..ef4a51565 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -92,8 +92,7 @@ class EtsFromJsonTest { @Test fun testLoadEtsFileAutoConvert() { val path = "/samples/source/example.ts" - val res = this::class.java.getResource(path)?.toURI()?.toPath() - ?: error("Resource not found: $path") + val res = getResourcePath(path) val etsFile = loadEtsFileAutoConvert(res) println("etsFile = $etsFile") } From eb27cabca76b71beb131588f99356420eca030fd Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 28 Oct 2024 18:12:58 +0300 Subject: [PATCH 083/120] Use AA's type inference once by default --- .../src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index 3180863d5..3e9196088 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -92,7 +92,7 @@ fun loadEtsFileAutoConvert(path: Path): EtsFile { val irFilePath = generateEtsIR( path, isProject = false, - useArkAnalyzerTypeInference = 2 + useArkAnalyzerTypeInference = 1, ) irFilePath.inputStream().use { stream -> val etsFileDto = EtsFileDto.loadFromJson(stream) @@ -104,12 +104,13 @@ fun loadEtsFileAutoConvert(path: Path): EtsFile { fun loadEtsProjectAutoConvert( path: Path, loadEntrypoints: Boolean = false, + useArkAnalyzerTypeInference: Int? = 1, ): EtsScene { val irFolderPath = generateEtsIR( path, isProject = true, loadEntrypoints = loadEntrypoints, - useArkAnalyzerTypeInference = 2, + useArkAnalyzerTypeInference = useArkAnalyzerTypeInference, ) val files = irFolderPath .walk() From a2667f87bc53e500cfd9e92d6ddb241af47164bb Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 30 Oct 2024 15:27:33 +0300 Subject: [PATCH 084/120] Make file signature non-null --- .../src/main/kotlin/org/jacodb/ets/dto/Convert.kt | 6 +++--- .../src/main/kotlin/org/jacodb/ets/dto/Signatures.kt | 4 ++-- .../main/kotlin/org/jacodb/ets/model/EtsSignature.kt | 12 ++++++++---- .../kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt | 6 ++++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index bdea2b544..0d026b6b0 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -558,7 +558,7 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { val superClassSignature = classDto.superClassName?.takeIf { it != "" }?.let { name -> EtsClassSignature( name = name, - file = null, // TODO + file = EtsFileSignature.EMPTY, // TODO namespace = null, // TODO ) } @@ -699,7 +699,7 @@ fun convertToEtsFileSignature(file: FileSignatureDto): EtsFileSignature { fun convertToEtsNamespaceSignature(namespace: NamespaceSignatureDto): EtsNamespaceSignature { return EtsNamespaceSignature( name = namespace.name, - file = namespace.declaringFile?.let { convertToEtsFileSignature(it) }, + file = namespace.declaringFile.let { convertToEtsFileSignature(it) }, namespace = namespace.declaringNamespace?.let { convertToEtsNamespaceSignature(it) }, ) } @@ -707,7 +707,7 @@ fun convertToEtsNamespaceSignature(namespace: NamespaceSignatureDto): EtsNamespa fun convertToEtsClassSignature(clazz: ClassSignatureDto): EtsClassSignature { return EtsClassSignature( name = clazz.name, - file = clazz.declaringFile?.let { convertToEtsFileSignature(it) }, + file = clazz.declaringFile.let { convertToEtsFileSignature(it) }, namespace = clazz.declaringNamespace?.let { convertToEtsNamespaceSignature(it) }, ) } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt index a136bc685..5484e65bf 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt @@ -31,7 +31,7 @@ data class FileSignatureDto( @Serializable data class NamespaceSignatureDto( val name: String, - val declaringFile: FileSignatureDto? = null, + val declaringFile: FileSignatureDto, val declaringNamespace: NamespaceSignatureDto? = null, ) { override fun toString(): String { @@ -48,7 +48,7 @@ data class NamespaceSignatureDto( @Serializable data class ClassSignatureDto( val name: String, - val declaringFile: FileSignatureDto? = null, + val declaringFile: FileSignatureDto, val declaringNamespace: NamespaceSignatureDto? = null, ) { override fun toString(): String { diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt index e91d0a566..2a44210b7 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt @@ -39,15 +39,19 @@ data class EtsFileSignature( // } return tmp } + + companion object { + val EMPTY = EtsFileSignature("", "") + } } data class EtsNamespaceSignature( val name: String, - val file: EtsFileSignature? = null, + val file: EtsFileSignature, val namespace: EtsNamespaceSignature? = null, ) { val enclosingFile: EtsFileSignature? - get() = file ?: namespace?.enclosingFile + get() = namespace?.enclosingFile ?: file override fun toString(): String { // TODO: 'file' is not included in the toString() output, @@ -62,7 +66,7 @@ data class EtsNamespaceSignature( data class EtsClassSignature( val name: String, - val file: EtsFileSignature? = null, + val file: EtsFileSignature, val namespace: EtsNamespaceSignature? = null, ) { // TODO: more manual testing is required in order to understand whether @@ -75,7 +79,7 @@ data class EtsClassSignature( // } val enclosingFile: EtsFileSignature? - get() = file ?: namespace?.enclosingFile + get() = namespace?.enclosingFile ?: file override fun toString(): String { return if (namespace != null) { diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index ef4a51565..d8419b629 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -43,6 +43,7 @@ import org.jacodb.ets.dto.TypeDto import org.jacodb.ets.dto.convertToEtsFile import org.jacodb.ets.dto.convertToEtsMethod import org.jacodb.ets.model.EtsClassSignature +import org.jacodb.ets.model.EtsFileSignature import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.test.utils.getResourcePath import org.jacodb.ets.test.utils.getResourcePathOrNull @@ -71,7 +72,7 @@ class EtsFromJsonTest { } private val defaultSignature = EtsMethodSignature( - enclosingClass = EtsClassSignature(name = "_DEFAULT_ARK_CLASS"), + enclosingClass = EtsClassSignature(name = "_DEFAULT_ARK_CLASS", file = EtsFileSignature.EMPTY), name = "_DEFAULT_ARK_METHOD", parameters = emptyList(), returnType = EtsAnyType, @@ -139,7 +140,7 @@ class EtsFromJsonTest { return@testFactory } container("load ${availableProjectNames.size} projects") { - for (projectName in availableProjectNames) { + for (projectName in availableProjectNames.sorted()) { test("load $projectName") { dynamicLoadEtsProject(projectName) } @@ -277,6 +278,7 @@ class EtsFromJsonTest { EtsMethodSignature( enclosingClass = EtsClassSignature( name = "_DEFAULT_ARK_CLASS", + file = EtsFileSignature.EMPTY, ), name = "_DEFAULT_ARK_METHOD", parameters = emptyList(), From 7ce9f3e684cb5f315b682ff0dd0f0b019684b4b4 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 31 Oct 2024 17:29:15 +0300 Subject: [PATCH 085/120] Handle decorators, modifiers, type parameters, etc --- .github/workflows/build-and-test.yml | 2 +- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 54 +++++++++++++---- .../main/kotlin/org/jacodb/ets/dto/Model.kt | 40 ++++++------- .../kotlin/org/jacodb/ets/dto/Serializers.kt | 35 ----------- .../org/jacodb/ets/model/EtsBaseModel.kt | 41 +++++++++++++ .../kotlin/org/jacodb/ets/model/EtsClass.kt | 8 ++- .../org/jacodb/ets/model/EtsDecorator.kt | 23 +++++++ .../kotlin/org/jacodb/ets/model/EtsField.kt | 11 +--- .../kotlin/org/jacodb/ets/model/EtsMethod.kt | 28 ++++----- .../org/jacodb/ets/model/EtsModifier.kt | 60 +++++++++++++++++++ .../org/jacodb/ets/utils/EtsFileDtoToDot.kt | 14 ++++- .../main/kotlin/org/jacodb/ets/utils/Utils.kt | 13 ++++ .../org/jacodb/ets/test/EtsFromJsonTest.kt | 56 +++++++---------- .../org/jacodb/ets/test/EtsSceneTest.kt | 1 + 14 files changed, 251 insertions(+), 135 deletions(-) create mode 100644 jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsBaseModel.kt create mode 100644 jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsDecorator.kt create mode 100644 jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsModifier.kt diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index af903b005..ebc35b996 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-10-28" + BRANCH="neo/2024-10-31" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 0d026b6b0..a876778d9 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -111,6 +111,7 @@ import org.jacodb.ets.graph.EtsCfg import org.jacodb.ets.model.EtsClass import org.jacodb.ets.model.EtsClassImpl import org.jacodb.ets.model.EtsClassSignature +import org.jacodb.ets.model.EtsDecorator import org.jacodb.ets.model.EtsField import org.jacodb.ets.model.EtsFieldImpl import org.jacodb.ets.model.EtsFieldSignature @@ -122,15 +123,18 @@ import org.jacodb.ets.model.EtsMethodImpl import org.jacodb.ets.model.EtsMethodParameter import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.model.EtsMethodSubSignature +import org.jacodb.ets.model.EtsModifiers import org.jacodb.ets.model.EtsNamespace import org.jacodb.ets.model.EtsNamespaceSignature class EtsMethodBuilder( signature: EtsMethodSignature, + typeParameters: List = emptyList(), locals: List = emptyList(), - modifiers: List = emptyList(), + modifiers: EtsModifiers = EtsModifiers.EMPTY, + decorators: List = emptyList(), ) { - val etsMethod = EtsMethodImpl(signature, locals, modifiers = modifiers) + private val etsMethod = EtsMethodImpl(signature, typeParameters, locals, modifiers, decorators) private val currentStmts: MutableList = mutableListOf() @@ -548,7 +552,8 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { ) return MethodDto( signature = signature, - modifiers = emptyList(), + modifiers = 0, + decorators = emptyList(), typeParameters = emptyList(), body = body, ) @@ -572,12 +577,20 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { val methods = methodDtos.map { convertToEtsMethod(it) } val ctor = convertToEtsMethod(ctorDto) + val typeParameters = classDto.typeParameters.map { convertToEtsType(it) } + + val modifiers = EtsModifiers(classDto.modifiers) + val decorators = classDto.decorators.map { convertToEtsDecorator(it) } + return EtsClassImpl( signature = signature, fields = fields, methods = methods, ctor = ctor, superClass = superClassSignature, + typeParameters = typeParameters, + modifiers = modifiers, + decorators = decorators, ) } @@ -742,17 +755,29 @@ fun convertToEtsMethodSignature(method: MethodSignatureDto): EtsMethodSignature fun convertToEtsMethod(method: MethodDto): EtsMethod { val signature = convertToEtsMethodSignature(method.signature) - val modifiers = method.modifiers - .filterIsInstance() - .map { it.modifier } + val typeParameters = method.typeParameters.map { convertToEtsType(it) } + val modifiers = EtsModifiers(method.modifiers) + val decorators = method.decorators.map { convertToEtsDecorator(it) } if (method.body != null) { val locals = method.body.locals.map { convertToEtsLocal(it) } - val builder = EtsMethodBuilder(signature, locals, modifiers) + val builder = EtsMethodBuilder( + signature = signature, + typeParameters = typeParameters, + locals = locals, + modifiers = modifiers, + decorators = decorators, + ) return builder.build(method.body.cfg) } else { - return EtsMethodImpl(signature, locals = emptyList(), modifiers = modifiers) + return EtsMethodImpl( + signature = signature, + typeParameters = typeParameters, + locals = emptyList(), + modifiers = modifiers, + decorators = decorators, + ) } } @@ -765,10 +790,7 @@ fun convertToEtsField(field: FieldDto): EtsField { type = convertToEtsType(field.signature.type), ) ), - modifiers = field.modifiers - ?.filterIsInstance() - ?.map { it.modifier } - .orEmpty(), + modifiers = EtsModifiers(field.modifiers), isOptional = field.isOptional, isDefinitelyAssigned = field.isDefinitelyAssigned, ) @@ -796,6 +818,14 @@ fun convertToEtsFile(file: EtsFileDto): EtsFile { ) } +fun convertToEtsDecorator(decorator: DecoratorDto): EtsDecorator { + return EtsDecorator( + name = decorator.kind, + // TODO: content + // TODO: param + ) +} + fun convertToEtsLocal(local: LocalDto): EtsLocal { return EtsLocal( name = local.name, diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt index 18f9117af..c6978c0a9 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt @@ -54,7 +54,8 @@ data class NamespaceDto( @Serializable data class ClassDto( val signature: ClassSignatureDto, - val modifiers: List, + val modifiers: Int, + val decorators: List, val typeParameters: List, val superClassName: String?, val implementedInterfaceNames: List, @@ -65,29 +66,17 @@ data class ClassDto( @Serializable data class FieldDto( val signature: FieldSignatureDto, - val typeParameters: List? = null, // TODO: remove - val modifiers: List? = null, - @SerialName("questionToken") val isOptional: Boolean = false, // '?' - @SerialName("exclamationToken") val isDefinitelyAssigned: Boolean = false, // '!' + val modifiers: Int, + val decorators: List, + @SerialName("questionToken") val isOptional: Boolean, // '?' + @SerialName("exclamationToken") val isDefinitelyAssigned: Boolean, // '!' ) -@Serializable(with = ModifierSerializer::class) -sealed class ModifierDto { - @Serializable - data class DecoratorItem( - val kind: String, - val content: String? = null, - val param: String? = null, - ) : ModifierDto() - - @Serializable - data class StringItem(val modifier: String) : ModifierDto() -} - @Serializable data class MethodDto( val signature: MethodSignatureDto, - val modifiers: List, + val modifiers: Int, + val decorators: List, val typeParameters: List, val body: BodyDto? = null, ) @@ -104,7 +93,8 @@ data class ImportInfoDto( val importType: String, val importFrom: String, val nameBeforeAs: String? = null, - val modifiers: List, + val modifiers: Int, + // val decorators: List, val originTsPosition: LineColPositionDto? = null, ) @@ -115,10 +105,18 @@ data class ExportInfoDto( val exportFrom: String? = null, val nameBeforeAs: String? = null, val isDefault: Boolean, - val modifiers: List, + val modifiers: Int, + // val decorators: List, val originTsPosition: LineColPositionDto? = null, ) +@Serializable +data class DecoratorDto( + val kind: String, + // val content: String? = null, + // val param: String? = null, +) + @Serializable data class LineColPositionDto( val line: Int, diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt index 88a20dadb..38763bd27 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Serializers.kt @@ -19,53 +19,18 @@ package org.jacodb.ets.dto import kotlinx.serialization.ExperimentalSerializationApi -import kotlinx.serialization.InternalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.SerializationException -import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.PrimitiveKind import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.descriptors.buildSerialDescriptor -import kotlinx.serialization.descriptors.element import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.encoding.Encoder import kotlinx.serialization.json.JsonDecoder import kotlinx.serialization.json.JsonEncoder -import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.booleanOrNull -import kotlinx.serialization.json.decodeFromJsonElement import kotlinx.serialization.json.double -import kotlinx.serialization.json.encodeToJsonElement -import kotlinx.serialization.json.intOrNull - -object ModifierSerializer : KSerializer { - @OptIn(InternalSerializationApi::class) - override val descriptor: SerialDescriptor = - buildSerialDescriptor("Modifier", PolymorphicKind.SEALED) { - element("DecoratorItem") - element("StringItem") - } - - override fun serialize(encoder: Encoder, value: ModifierDto) { - require(encoder is JsonEncoder) - when (value) { - is ModifierDto.DecoratorItem -> encoder.encodeJsonElement(encoder.json.encodeToJsonElement(value)) - is ModifierDto.StringItem -> encoder.encodeString(value.modifier) - } - } - - override fun deserialize(decoder: Decoder): ModifierDto { - require(decoder is JsonDecoder) - val element = decoder.decodeJsonElement() - return when { - element is JsonObject -> decoder.json.decodeFromJsonElement(element) - element is JsonPrimitive && element.isString -> ModifierDto.StringItem(element.content) - else -> throw SerializationException("Unsupported modifier: $element") - } - } -} object PrimitiveLiteralSerializer : KSerializer { override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("PrimitiveLiteral", PrimitiveKind.STRING) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsBaseModel.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsBaseModel.kt new file mode 100644 index 000000000..731e89982 --- /dev/null +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsBaseModel.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.ets.model + +interface EtsBaseModel { + val modifiers: EtsModifiers + val decorators: List + + val isPrivate: Boolean get() = modifiers.isPrivate + val isProtected: Boolean get() = modifiers.isProtected + val isPublic: Boolean get() = modifiers.isPublic + val isExport: Boolean get() = modifiers.isExport + val isStatic: Boolean get() = modifiers.isStatic + val isAbstract: Boolean get() = modifiers.isAbstract + val isAsync: Boolean get() = modifiers.isAsync + val isConst: Boolean get() = modifiers.isConst + val isAccessor: Boolean get() = modifiers.isAccessor + val isDefault: Boolean get() = modifiers.isDefault + val isIn: Boolean get() = modifiers.isIn + val isReadonly: Boolean get() = modifiers.isReadonly + val isOut: Boolean get() = modifiers.isOut + val isOverride: Boolean get() = modifiers.isOverride + val isDeclare: Boolean get() = modifiers.isDeclare + + fun hasModifier(modifier: EtsModifier): Boolean = modifiers.hasModifier(modifier) + fun hasDecorator(decorator: EtsDecorator): Boolean = decorators.contains(decorator) +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt index d8661f88c..a6e9fc0cf 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt @@ -16,8 +16,11 @@ package org.jacodb.ets.model -interface EtsClass { +import org.jacodb.ets.base.EtsType + +interface EtsClass : EtsBaseModel { val signature: EtsClassSignature + val typeParameters: List val fields: List val methods: List val ctor: EtsMethod @@ -33,6 +36,9 @@ class EtsClassImpl( override val methods: List, override val ctor: EtsMethod, override val superClass: EtsClassSignature? = null, + override val typeParameters: List = emptyList(), + override val modifiers: EtsModifiers = EtsModifiers.EMPTY, + override val decorators: List = emptyList(), ) : EtsClass { init { require(ctor !in methods) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsDecorator.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsDecorator.kt new file mode 100644 index 000000000..908628a81 --- /dev/null +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsDecorator.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.ets.model + +data class EtsDecorator( + val name: String, // kind + // TODO: content + // TODO: param +) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsField.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsField.kt index 47ac1d67d..326faf74d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsField.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsField.kt @@ -33,8 +33,7 @@ interface EtsField { class EtsFieldImpl( override val signature: EtsFieldSignature, - val accessFlags: AccessFlags = AccessFlags(), - val modifiers: List = emptyList(), + val modifiers: EtsModifiers = EtsModifiers.EMPTY, val isOptional: Boolean = false, // '?' val isDefinitelyAssigned: Boolean = false, // '!' ) : EtsField { @@ -45,11 +44,3 @@ class EtsFieldImpl( return signature.toString() } } - -data class AccessFlags( - var isStatic: Boolean = false, - var isPublic: Boolean = false, - var isPrivate: Boolean = false, - var isProtected: Boolean = false, - var isReadOnly: Boolean = false, -) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt index 47fb7de74..fb294301c 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsMethod.kt @@ -14,6 +14,8 @@ * limitations under the License. */ +@file:Suppress("PropertyName") + package org.jacodb.ets.model import org.jacodb.api.common.CommonMethod @@ -21,29 +23,19 @@ import org.jacodb.ets.base.EtsLocal import org.jacodb.ets.base.EtsType import org.jacodb.ets.graph.EtsCfg -// TODO: decorators // TODO: typeParameters -interface EtsMethod : CommonMethod { +interface EtsMethod : EtsBaseModel, CommonMethod { val signature: EtsMethodSignature - val cfg: EtsCfg + val typeParameters: List val locals: List - val modifiers: List + val cfg: EtsCfg val enclosingClass: EtsClassSignature get() = signature.enclosingClass - val isStatic: Boolean - get() = modifiers.contains("StaticKeyword") - - val isPrivate: Boolean - get() = modifiers.contains("PrivateKeyword") - // If not specified, entity is public if not private and not protected - val isPublic: Boolean - get() = modifiers.contains("PublicKeyword") || (!isPrivate && !isProtected) - - val isProtected: Boolean - get() = modifiers.contains("ProtectedKeyword") + override val isPublic: Boolean + get() = super.isPublic || (!isPrivate && !isProtected) override val name: String get() = signature.name @@ -61,10 +53,12 @@ interface EtsMethod : CommonMethod { class EtsMethodImpl( override val signature: EtsMethodSignature, + override val typeParameters: List = emptyList(), override val locals: List = emptyList(), - override val modifiers: List = emptyList(), + override val modifiers: EtsModifiers = EtsModifiers.EMPTY, + override val decorators: List = emptyList(), ) : EtsMethod { - internal var _cfg: EtsCfg? = null + var _cfg: EtsCfg? = null override val cfg: EtsCfg get() = _cfg ?: EtsCfg.empty() diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsModifier.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsModifier.kt new file mode 100644 index 000000000..cf584b780 --- /dev/null +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsModifier.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.ets.model + +enum class EtsModifier(val value: Int, val string: String) { + PRIVATE(1 shl 0, "private"), + PROTECTED(1 shl 1, "protected"), + PUBLIC(1 shl 2, "public"), + EXPORT(1 shl 3, "export"), + STATIC(1 shl 4, "static"), + ABSTRACT(1 shl 5, "abstract"), + ASYNC(1 shl 6, "async"), + CONST(1 shl 7, "const"), + ACCESSOR(1 shl 8, "accessor"), + DEFAULT(1 shl 9, "default"), + IN(1 shl 10, "in"), + READONLY(1 shl 11, "readonly"), + OUT(1 shl 12, "out"), + OVERRIDE(1 shl 13, "override"), + DECLARE(1 shl 14, "declare"); +} + +@JvmInline +value class EtsModifiers(val mask: Int) { + companion object { + val EMPTY = EtsModifiers(0) + } + + val isPrivate: Boolean get() = hasModifier(EtsModifier.PRIVATE) + val isProtected: Boolean get() = hasModifier(EtsModifier.PROTECTED) + val isPublic: Boolean get() = hasModifier(EtsModifier.PUBLIC) + val isExport: Boolean get() = hasModifier(EtsModifier.EXPORT) + val isStatic: Boolean get() = hasModifier(EtsModifier.STATIC) + val isAbstract: Boolean get() = hasModifier(EtsModifier.ABSTRACT) + val isAsync: Boolean get() = hasModifier(EtsModifier.ASYNC) + val isConst: Boolean get() = hasModifier(EtsModifier.CONST) + val isAccessor: Boolean get() = hasModifier(EtsModifier.ACCESSOR) + val isDefault: Boolean get() = hasModifier(EtsModifier.DEFAULT) + val isIn: Boolean get() = hasModifier(EtsModifier.IN) + val isReadonly: Boolean get() = hasModifier(EtsModifier.READONLY) + val isOut: Boolean get() = hasModifier(EtsModifier.OUT) + val isOverride: Boolean get() = hasModifier(EtsModifier.OVERRIDE) + val isDeclare: Boolean get() = hasModifier(EtsModifier.DECLARE) + + fun hasModifier(modifier: EtsModifier): Boolean = (mask and modifier.value) != 0 +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt index e00dda059..1587056f7 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt @@ -46,10 +46,20 @@ fun EtsFileDto.toDot(useLR: Boolean = false): String { fun classLabel(clazz: ClassDto): String { val labelLines: MutableList = mutableListOf() - labelLines += clazz.signature.name + run { + val name = clazz.signature.name + val generics = if (clazz.typeParameters.isNotEmpty()) { + "<${clazz.typeParameters.joinToString()}>" + } else { + "" + } + labelLines += "$name$generics" + } labelLines += "Fields: (${clazz.fields.size})" clazz.fields.forEach { field -> - labelLines += " ${field.signature.name}: ${field.signature.type}" + val name = field.signature.name + val returnType = field.signature.type + labelLines += " $name: $returnType" } labelLines += "Methods: (${clazz.methods.size})" clazz.methods.forEach { method -> diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt index 14e525564..380772e30 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Utils.kt @@ -72,16 +72,20 @@ fun EtsFileDto.toText(): String { lines += " superClass = '${clazz.superClassName}'" lines += " typeParameters = ${clazz.typeParameters}" lines += " modifiers = ${clazz.modifiers}" + lines += " decorators = ${clazz.decorators}" lines += " fields: ${clazz.fields.size}" clazz.fields.forEach { field -> lines += " - FIELD '${field.signature}'" lines += " modifiers = ${field.modifiers}" + lines += " decorators = ${field.decorators}" lines += " isOptional = ${field.isOptional}" lines += " isDefinitelyAssigned = ${field.isDefinitelyAssigned}" } lines += " methods: ${clazz.methods.size}" clazz.methods.forEach { method -> lines += " - METHOD '${method.signature}'" + lines += " modifiers = ${method.modifiers}" + lines += " decorators = ${method.decorators}" lines += " typeParameters = ${method.typeParameters}" if (method.body != null) { lines += " locals = ${method.body.locals}" @@ -106,12 +110,18 @@ fun EtsFile.toText(): String { lines += "EtsFile '${signature}':" classes.forEach { clazz -> lines += "= CLASS '${clazz.signature}':" + lines += " typeParameters = ${clazz.typeParameters}" + lines += " modifiers = ${clazz.modifiers}" + lines += " decorators = ${clazz.decorators}" lines += " superClass = '${clazz.superClass}'" lines += " fields: ${clazz.fields.size}" clazz.fields.forEach { field -> lines += " - FIELD '${field.signature}'" } lines += " constructor = '${clazz.ctor.signature}'" + lines += " typeParameters = ${clazz.ctor.typeParameters}" + lines += " modifiers = ${clazz.ctor.modifiers}" + lines += " decorators = ${clazz.ctor.decorators}" lines += " locals: ${clazz.ctor.locals.size}" lines += " stmts: ${clazz.ctor.cfg.stmts.size}" clazz.ctor.cfg.stmts.forEach { stmt -> @@ -122,6 +132,9 @@ fun EtsFile.toText(): String { lines += " methods: ${clazz.methods.size}" clazz.methods.forEach { method -> lines += " - METHOD '${method.signature}':" + lines += " typeParameters = ${method.typeParameters}" + lines += " modifiers = ${method.modifiers}" + lines += " decorators = ${method.decorators}" lines += " locals: ${method.locals.size}" lines += " stmts: ${method.cfg.stmts.size}" method.cfg.stmts.forEach { stmt -> diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index d8419b629..97008805e 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -27,6 +27,7 @@ import org.jacodb.ets.base.EtsReturnStmt import org.jacodb.ets.base.EtsUnknownType import org.jacodb.ets.dto.AnyTypeDto import org.jacodb.ets.dto.ClassSignatureDto +import org.jacodb.ets.dto.DecoratorDto import org.jacodb.ets.dto.EtsMethodBuilder import org.jacodb.ets.dto.FieldDto import org.jacodb.ets.dto.FieldSignatureDto @@ -34,7 +35,6 @@ import org.jacodb.ets.dto.FileSignatureDto import org.jacodb.ets.dto.LiteralTypeDto import org.jacodb.ets.dto.LocalDto import org.jacodb.ets.dto.MethodDto -import org.jacodb.ets.dto.ModifierDto import org.jacodb.ets.dto.NumberTypeDto import org.jacodb.ets.dto.PrimitiveLiteralDto import org.jacodb.ets.dto.ReturnVoidStmtDto @@ -209,7 +209,8 @@ class EtsFromJsonTest { name = "x", type = NumberTypeDto, ), - modifiers = emptyList(), + modifiers = 0, + decorators = emptyList(), isOptional = true, isDefinitelyAssigned = false, ) @@ -241,7 +242,11 @@ class EtsFromJsonTest { { "signature": { "declaringClass": { - "name": "_DEFAULT_ARK_CLASS" + "name": "_DEFAULT_ARK_CLASS", + "declaringFile": { + "projectName": "TestProject", + "fileName": "test.ts" + } }, "name": "_DEFAULT_ARK_METHOD", "parameters": [], @@ -249,7 +254,8 @@ class EtsFromJsonTest { "_": "UnknownType" } }, - "modifiers": [], + "modifiers": 0, + "decorators": [], "typeParameters": [], "body": { "locals": [], @@ -278,7 +284,10 @@ class EtsFromJsonTest { EtsMethodSignature( enclosingClass = EtsClassSignature( name = "_DEFAULT_ARK_CLASS", - file = EtsFileSignature.EMPTY, + file = EtsFileSignature( + projectName = "TestProject", + fileName = "test.ts", + ), ), name = "_DEFAULT_ARK_METHOD", parameters = emptyList(), @@ -297,44 +306,19 @@ class EtsFromJsonTest { } @Test - fun testLoadModifierFromJson() { + fun testLoadDecoratorFromJson() { val jsonString = """ { - "kind": "cat", - "content": "Brian" + "kind": "cat" } """.trimIndent() - val modifierDto = Json.decodeFromString(jsonString) - println("modifierDto = $modifierDto") - Assertions.assertEquals(ModifierDto.DecoratorItem("cat", "Brian"), modifierDto) - val jsonString2 = json.encodeToString(modifierDto) + val decoratorDto = Json.decodeFromString(jsonString) + println("decoratorDto = $decoratorDto") + Assertions.assertEquals(DecoratorDto("cat"), decoratorDto) + val jsonString2 = json.encodeToString(decoratorDto) println("json: $jsonString2") } - @Test - fun testLoadListOfModifiersFromJson() { - val jsonString = """ - [ - { - "kind": "cat", - "content": "Bruce" - }, - "public", - "static" - ] - """.trimIndent() - val modifiers = Json.decodeFromString>(jsonString) - println("modifiers = $modifiers") - Assertions.assertEquals( - listOf( - ModifierDto.DecoratorItem("cat", "Bruce"), - ModifierDto.StringItem("public"), - ModifierDto.StringItem("static"), - ), - modifiers - ) - } - @Test fun testLoadLiteralTypeFromJson() { // TS: `let x: "hello" = "hello";` diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt index 9c68b0737..90146c5d5 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt @@ -38,6 +38,7 @@ import org.jacodb.ets.model.EtsFile import org.jacodb.ets.model.EtsFileSignature import org.jacodb.ets.model.EtsMethodImpl import org.jacodb.ets.model.EtsMethodSignature +import org.jacodb.ets.model.EtsModifiers import org.jacodb.ets.model.EtsScene import kotlin.test.Test import kotlin.test.assertEquals From 1b7b702e8fe69b398078887c279f370183c54869 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 12:15:43 +0300 Subject: [PATCH 086/120] Add constants --- .../org/jacodb/analysis/util/EtsTraits.kt | 3 +- .../kotlin/org/jacodb/ets/base/Constants.kt | 32 +++++++++++++++++++ .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 9 +++--- .../jacodb/ets/graph/EtsApplicationGraph.kt | 6 ++-- .../kotlin/org/jacodb/ets/test/EtsFileTest.kt | 6 ++-- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 17 ++++++---- .../org/jacodb/ets/test/EtsSceneTest.kt | 5 +-- 7 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt index eb11c3d10..3601d92d0 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt @@ -25,6 +25,7 @@ import org.jacodb.api.common.cfg.CommonArgument import org.jacodb.api.common.cfg.CommonCallExpr import org.jacodb.api.common.cfg.CommonExpr import org.jacodb.api.common.cfg.CommonValue +import org.jacodb.ets.base.CONSTRUCTOR_NAME import org.jacodb.ets.base.EtsAnyType import org.jacodb.ets.base.EtsArrayAccess import org.jacodb.ets.base.EtsCallExpr @@ -71,7 +72,7 @@ interface EtsTraits : Traits { get() = EtsThis(EtsClassType(enclosingClass)) override val EtsMethod.isConstructor: Boolean - get() = name == "constructor" + get() = name == CONSTRUCTOR_NAME override fun CommonExpr.toPathOrNull(): AccessPath? { check(this is EtsEntity) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt new file mode 100644 index 000000000..42ffa7488 --- /dev/null +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.ets.base + +const val CONSTRUCTOR_NAME = "constructor" + +const val DEFAULT_ARK_CLASS_NAME = "%dflt" +const val DEFAULT_ARK_METHOD_NAME = "%dflt" + +const val UNKNOWN_FILE_NAME = "%unk" + +const val INSTANCE_INIT_METHOD_NAME = "%instInit" +const val STATIC_INIT_METHOD_NAME = "%statInit" + +const val ANONYMOUS_CLASS_PREFIX = "%AC" +const val ANONYMOUS_METHOD_PREFIX = "%AM" + +const val TEMP_LOCAL_PREFIX = "%" diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index a876778d9..53a9de71e 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -16,6 +16,7 @@ package org.jacodb.ets.dto +import org.jacodb.ets.base.CONSTRUCTOR_NAME import org.jacodb.ets.base.EtsAddExpr import org.jacodb.ets.base.EtsAndExpr import org.jacodb.ets.base.EtsAnyType @@ -539,14 +540,14 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { successors = emptyList(), predecessors = emptyList(), stmts = listOf( - ReturnVoidStmtDto - ) + ReturnVoidStmtDto, + ), ) val cfg = CfgDto(blocks = listOf(zeroBlock)) val body = BodyDto(locals = emptyList(), cfg = cfg) val signature = MethodSignatureDto( declaringClass = classSignatureDto, - name = "constructor", + name = CONSTRUCTOR_NAME, parameters = emptyList(), returnType = ClassTypeDto(classSignatureDto), ) @@ -570,7 +571,7 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { val fields = classDto.fields.map { convertToEtsField(it) } - val (methodDtos, ctorDtos) = classDto.methods.partition { it.signature.name != "constructor" } + val (methodDtos, ctorDtos) = classDto.methods.partition { it.signature.name != CONSTRUCTOR_NAME } check(ctorDtos.size <= 1) { "Class should not have multiple constructors" } val ctorDto = ctorDtos.firstOrNull() ?: defaultConstructorDto(classDto.signature) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 1f7cd18bf..fb4071bc0 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -18,7 +18,9 @@ package org.jacodb.ets.graph import mu.KotlinLogging import org.jacodb.api.common.analysis.ApplicationGraph +import org.jacodb.ets.base.CONSTRUCTOR_NAME import org.jacodb.ets.base.EtsStmt +import org.jacodb.ets.base.UNKNOWN_FILE_NAME import org.jacodb.ets.model.EtsClass import org.jacodb.ets.model.EtsClassSignature import org.jacodb.ets.model.EtsFileSignature @@ -35,7 +37,7 @@ interface EtsApplicationGraph : ApplicationGraph { } private fun EtsFileSignature?.isUnknown(): Boolean = - this == null || fileName.isBlank() || fileName == "_UnknownFileName" + this == null || fileName.isBlank() || fileName == UNKNOWN_FILE_NAME private fun EtsClassSignature.isUnknown(): Boolean = name.isBlank() @@ -121,7 +123,7 @@ class EtsApplicationGraphImpl( } // Note: specific resolve for constructor: - if (callee.name == "constructor") { + if (callee.name == CONSTRUCTOR_NAME) { if (!callee.enclosingClass.isIdeal()) { // Constructor signature is garbage. Sorry, can't do anything in such case. return emptySequence() diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt index d339009f7..d3ec69b53 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt @@ -22,6 +22,8 @@ import org.jacodb.ets.base.EtsLocal import org.jacodb.ets.base.EtsNumberConstant import org.jacodb.ets.base.EtsReturnStmt import org.jacodb.ets.base.EtsThis +import org.jacodb.ets.base.INSTANCE_INIT_METHOD_NAME +import org.jacodb.ets.base.STATIC_INIT_METHOD_NAME import org.jacodb.ets.model.EtsFile import org.jacodb.ets.test.utils.loadEtsFileFromResource import kotlin.test.Test @@ -82,7 +84,7 @@ class EtsFileTest { // instance initializer run { - val method = cls.methods.single { it.name == "@instance_init" } + val method = cls.methods.single { it.name == INSTANCE_INIT_METHOD_NAME } assertEquals(3, method.cfg.instructions.size) // Local("this") := ThisRef @@ -129,7 +131,7 @@ class EtsFileTest { // static initializer run { - val method = cls.methods.single { it.name == "@static_init" } + val method = cls.methods.single { it.name == STATIC_INIT_METHOD_NAME } assertEquals(3, method.cfg.instructions.size) // Local("this") := ThisRef diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 97008805e..205aff1a6 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -20,6 +20,8 @@ import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import mu.KotlinLogging +import org.jacodb.ets.base.DEFAULT_ARK_CLASS_NAME +import org.jacodb.ets.base.DEFAULT_ARK_METHOD_NAME import org.jacodb.ets.base.EtsAnyType import org.jacodb.ets.base.EtsInstLocation import org.jacodb.ets.base.EtsLocal @@ -72,8 +74,11 @@ class EtsFromJsonTest { } private val defaultSignature = EtsMethodSignature( - enclosingClass = EtsClassSignature(name = "_DEFAULT_ARK_CLASS", file = EtsFileSignature.EMPTY), - name = "_DEFAULT_ARK_METHOD", + enclosingClass = EtsClassSignature( + name = DEFAULT_ARK_CLASS_NAME, + file = EtsFileSignature.EMPTY, + ), + name = DEFAULT_ARK_METHOD_NAME, parameters = emptyList(), returnType = EtsAnyType, ) @@ -242,13 +247,13 @@ class EtsFromJsonTest { { "signature": { "declaringClass": { - "name": "_DEFAULT_ARK_CLASS", + "name": "$DEFAULT_ARK_CLASS_NAME", "declaringFile": { "projectName": "TestProject", "fileName": "test.ts" } }, - "name": "_DEFAULT_ARK_METHOD", + "name": "$DEFAULT_ARK_METHOD_NAME", "parameters": [], "returnType": { "_": "UnknownType" @@ -283,13 +288,13 @@ class EtsFromJsonTest { Assertions.assertEquals( EtsMethodSignature( enclosingClass = EtsClassSignature( - name = "_DEFAULT_ARK_CLASS", + name = DEFAULT_ARK_CLASS_NAME, file = EtsFileSignature( projectName = "TestProject", fileName = "test.ts", ), ), - name = "_DEFAULT_ARK_METHOD", + name = DEFAULT_ARK_METHOD_NAME, parameters = emptyList(), returnType = EtsUnknownType, ), diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt index 90146c5d5..d46afb53e 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsSceneTest.kt @@ -16,6 +16,7 @@ package org.jacodb.ets.test +import org.jacodb.ets.base.CONSTRUCTOR_NAME import org.jacodb.ets.base.EtsArrayType import org.jacodb.ets.base.EtsAssignStmt import org.jacodb.ets.base.EtsCallStmt @@ -76,7 +77,7 @@ class EtsSceneTest { val ctorCat = EtsMethodImpl( signature = EtsMethodSignature( enclosingClass = classCatSignature, - name = "constructor", + name = CONSTRUCTOR_NAME, parameters = emptyList(), returnType = EtsVoidType, ), @@ -152,7 +153,7 @@ class EtsSceneTest { val ctorBox = EtsMethodImpl( signature = EtsMethodSignature( enclosingClass = classBoxSignature, - name = "constructor", + name = CONSTRUCTOR_NAME, parameters = emptyList(), returnType = EtsVoidType, ), From 30941c98b6e48bb56f4b9d5e7c66345020dbdb03 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 12:20:41 +0300 Subject: [PATCH 087/120] Bump AA on CI --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ebc35b996..e2be50e21 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-10-31" + BRANCH="neo/2024-11-06" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break From 2f8b10bc1166bf1d25bf81fd8d9bf7860f54ffd2 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 15:45:40 +0300 Subject: [PATCH 088/120] Add more "unknown" constants --- jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt index 42ffa7488..1c6164190 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/Constants.kt @@ -21,7 +21,12 @@ const val CONSTRUCTOR_NAME = "constructor" const val DEFAULT_ARK_CLASS_NAME = "%dflt" const val DEFAULT_ARK_METHOD_NAME = "%dflt" +const val UNKNOWN_PROJECT_NAME = "%unk" const val UNKNOWN_FILE_NAME = "%unk" +const val UNKNOWN_NAMESPACE_NAME = "%unk" +const val UNKNOWN_CLASS_NAME = "" // TODO: consult AA/src/core/common/Const.ts +const val UNKNOWN_FIELD_NAME = "" // TODO: consult AA/src/core/common/Const.ts +const val UNKNOWN_METHOD_NAME = "" // TODO: consult AA/src/core/common/Const.ts const val INSTANCE_INIT_METHOD_NAME = "%instInit" const val STATIC_INIT_METHOD_NAME = "%statInit" From 416523c327145bdce6382009cbc090f51dd45d8d Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 15:46:33 +0300 Subject: [PATCH 089/120] Print classes and methods --- .../test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 205aff1a6..4c5bd7a02 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -93,6 +93,14 @@ class EtsFromJsonTest { println("etsDto = $etsDto") val ets = convertToEtsFile(etsDto) println("ets = $ets") + + println("Classes: ${ets.classes.size}") + for (cls in ets.classes) { + println("= $cls with ${cls.methods.size} methods:") + for (method in cls.methods) { + println(" - $method") + } + } } @Test From 8f2054f3cef319c8e749a2c1c19eee67098fbacd Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 15:46:58 +0300 Subject: [PATCH 090/120] Refine signatures --- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 26 +++---- .../kotlin/org/jacodb/ets/dto/Signatures.kt | 8 +-- .../org/jacodb/ets/model/EtsSignature.kt | 69 +++++-------------- .../org/jacodb/ets/test/EtsFromJsonTest.kt | 2 +- 4 files changed, 31 insertions(+), 74 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 53a9de71e..69728f74d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -123,7 +123,6 @@ import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsMethodImpl import org.jacodb.ets.model.EtsMethodParameter import org.jacodb.ets.model.EtsMethodSignature -import org.jacodb.ets.model.EtsMethodSubSignature import org.jacodb.ets.model.EtsModifiers import org.jacodb.ets.model.EtsNamespace import org.jacodb.ets.model.EtsNamespaceSignature @@ -564,8 +563,7 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { val superClassSignature = classDto.superClassName?.takeIf { it != "" }?.let { name -> EtsClassSignature( name = name, - file = EtsFileSignature.EMPTY, // TODO - namespace = null, // TODO + file = EtsFileSignature.DEFAULT, ) } @@ -739,18 +737,16 @@ fun convertToEtsFieldSignature(field: FieldSignatureDto): EtsFieldSignature { fun convertToEtsMethodSignature(method: MethodSignatureDto): EtsMethodSignature { return EtsMethodSignature( enclosingClass = convertToEtsClassSignature(method.declaringClass), - sub = EtsMethodSubSignature( - name = method.name, - parameters = method.parameters.mapIndexed { index, param -> - EtsMethodParameter( - index = index, - name = param.name, - type = convertToEtsType(param.type), - isOptional = param.isOptional - ) - }, - returnType = convertToEtsType(method.returnType), - ) + name = method.name, + parameters = method.parameters.mapIndexed { index, param -> + EtsMethodParameter( + index = index, + name = param.name, + type = convertToEtsType(param.type), + isOptional = param.isOptional + ) + }, + returnType = convertToEtsType(method.returnType), ) } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt index 5484e65bf..b8fbf624f 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt @@ -37,10 +37,8 @@ data class NamespaceSignatureDto( override fun toString(): String { return if (declaringNamespace != null) { "$declaringNamespace::$name" - } else if (declaringFile != null) { - "$name in $declaringFile" } else { - name + "$declaringFile::$name" } } } @@ -54,10 +52,8 @@ data class ClassSignatureDto( override fun toString(): String { return if (declaringNamespace != null) { "$declaringNamespace::$name" - } else if (declaringFile != null) { - "$name in $declaringFile" } else { - name + "$declaringFile::$name" } } } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt index 2a44210b7..a96cc460d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt @@ -18,6 +18,10 @@ package org.jacodb.ets.model import org.jacodb.api.common.CommonMethodParameter import org.jacodb.ets.base.EtsType +import org.jacodb.ets.base.UNKNOWN_CLASS_NAME +import org.jacodb.ets.base.UNKNOWN_FILE_NAME +import org.jacodb.ets.base.UNKNOWN_NAMESPACE_NAME +import org.jacodb.ets.base.UNKNOWN_PROJECT_NAME /** * Precompiled [Regex] for `.d.ts` and `.ts` file extensions. @@ -30,18 +34,12 @@ data class EtsFileSignature( ) { override fun toString(): String { // Remove ".d.ts" and ".ts" file ext: - val tmp = fileName.replace(REGEX_TS_SUFFIX, "") - // TODO: projectName was omitted for now in toString(), since it disturbs the debugging output. - // return if (projectName.isNotBlank()) { - // "@$projectName/$tmp" - // } else { - // tmp - // } - return tmp + val name = fileName.replace(REGEX_TS_SUFFIX, "") + return "@$projectName/$name" } companion object { - val EMPTY = EtsFileSignature("", "") + val DEFAULT = EtsFileSignature(projectName = UNKNOWN_PROJECT_NAME, fileName = UNKNOWN_FILE_NAME) } } @@ -50,18 +48,17 @@ data class EtsNamespaceSignature( val file: EtsFileSignature, val namespace: EtsNamespaceSignature? = null, ) { - val enclosingFile: EtsFileSignature? - get() = namespace?.enclosingFile ?: file - override fun toString(): String { - // TODO: 'file' is not included in the toString() output, - // because it only disturbs the debugging output. return if (namespace != null) { "$namespace::$name" } else { - name + "$file: $name" } } + + companion object { + val DEFAULT = EtsNamespaceSignature(name = UNKNOWN_NAMESPACE_NAME, file = EtsFileSignature.DEFAULT) + } } data class EtsClassSignature( @@ -69,27 +66,17 @@ data class EtsClassSignature( val file: EtsFileSignature, val namespace: EtsNamespaceSignature? = null, ) { - // TODO: more manual testing is required in order to understand whether - // the class can have both "declaring file" and "declaring namespace". - // Until then, the following check is commented out: - // init { - // require(!(file != null && namespace != null)) { - // "Class cannot have both declaring file and declaring namespace" - // } - // } - - val enclosingFile: EtsFileSignature? - get() = namespace?.enclosingFile ?: file - override fun toString(): String { return if (namespace != null) { "$namespace::$name" - } else if (file != null) { - "$name in $file" } else { - name + "$file: $name" } } + + companion object { + val DEFAULT = EtsClassSignature(name = UNKNOWN_CLASS_NAME, file = EtsFileSignature.DEFAULT) + } } data class EtsFieldSignature( @@ -122,34 +109,12 @@ data class EtsMethodSignature( val parameters: List, val returnType: EtsType, ) { - - constructor( - enclosingClass: EtsClassSignature, - sub: EtsMethodSubSignature, - ) : this( - enclosingClass, - sub.name, - sub.parameters, - sub.returnType, - ) - override fun toString(): String { val params = parameters.joinToString() return "${enclosingClass.name}::$name($params): $returnType" } } -data class EtsMethodSubSignature( - val name: String, - val parameters: List, - val returnType: EtsType, -) { - override fun toString(): String { - val params = parameters.joinToString() - return "$name($params): $returnType" - } -} - data class EtsMethodParameter( val index: Int, val name: String, diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 4c5bd7a02..7eae241b5 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -76,7 +76,7 @@ class EtsFromJsonTest { private val defaultSignature = EtsMethodSignature( enclosingClass = EtsClassSignature( name = DEFAULT_ARK_CLASS_NAME, - file = EtsFileSignature.EMPTY, + file = EtsFileSignature.DEFAULT, ), name = DEFAULT_ARK_METHOD_NAME, parameters = emptyList(), From dacf419988efb4d9b443227bc24183d65c5048b4 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 16:44:07 +0300 Subject: [PATCH 091/120] Extract implemented interfaces from DTO --- jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt | 7 +++++++ .../src/main/kotlin/org/jacodb/ets/model/EtsClass.kt | 2 ++ 2 files changed, 9 insertions(+) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 69728f74d..2ba9f7915 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -566,6 +566,12 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { file = EtsFileSignature.DEFAULT, ) } + val implementedInterfaces = classDto.implementedInterfaceNames.map { name -> + EtsClassSignature( + name = name, + file = EtsFileSignature.DEFAULT, + ) + } val fields = classDto.fields.map { convertToEtsField(it) } @@ -587,6 +593,7 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { methods = methods, ctor = ctor, superClass = superClassSignature, + implementedInterfaces = implementedInterfaces, typeParameters = typeParameters, modifiers = modifiers, decorators = decorators, diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt index a6e9fc0cf..560dac296 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsClass.kt @@ -25,6 +25,7 @@ interface EtsClass : EtsBaseModel { val methods: List val ctor: EtsMethod val superClass: EtsClassSignature? + val implementedInterfaces: List val name: String get() = signature.name @@ -36,6 +37,7 @@ class EtsClassImpl( override val methods: List, override val ctor: EtsMethod, override val superClass: EtsClassSignature? = null, + override val implementedInterfaces: List = emptyList(), override val typeParameters: List = emptyList(), override val modifiers: EtsModifiers = EtsModifiers.EMPTY, override val decorators: List = emptyList(), From 11c00ff0b72b5b93ef6be28a565916f181085c97 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 17:46:39 +0300 Subject: [PATCH 092/120] Infer types in generateTestResource --- jacodb-ets/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/build.gradle.kts b/jacodb-ets/build.gradle.kts index 48ad2553c..27a60787b 100644 --- a/jacodb-ets/build.gradle.kts +++ b/jacodb-ets/build.gradle.kts @@ -72,6 +72,7 @@ tasks.register("generateTestResources") { "--multi", inputDir.relativeTo(resources).path, outputDir.relativeTo(resources).path, + "-t", ) println("Running: '${cmd.joinToString(" ")}'") val process = ProcessBuilder(cmd).directory(resources).start() From 341a11015269e05fd9d075cfb3e4e6ccd29d7ab0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 6 Nov 2024 18:32:41 +0300 Subject: [PATCH 093/120] Add generics for ClassType, FunctionType, UnclearRefType --- .../kotlin/org/jacodb/ets/base/EtsType.kt | 29 ++++++++++++++++--- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 9 ++++-- .../kotlin/org/jacodb/ets/dto/Signatures.kt | 4 +-- .../main/kotlin/org/jacodb/ets/dto/Types.kt | 26 +++++++++++++++-- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt index 520df37c6..3c059ff91 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt @@ -217,10 +217,16 @@ data class EtsLiteralType( interface EtsRefType : EtsType data class EtsClassType( - val classSignature: EtsClassSignature, + val signature: EtsClassSignature, + val typeParameters: List = emptyList(), ) : EtsRefType { override val typeName: String - get() = classSignature.name + get() = if (typeParameters.isNotEmpty()) { + val generics = typeParameters.joinToString() + "${signature.name}<$generics>" + } else { + signature.name + } override fun toString(): String = typeName @@ -231,9 +237,15 @@ data class EtsClassType( data class EtsFunctionType( val method: EtsMethodSignature, + val typeParameters: List = emptyList(), ) : EtsRefType { override val typeName: String - get() = method.name + get() = if (typeParameters.isNotEmpty()) { + val generics = typeParameters.joinToString() + "${method.name}<$generics>" + } else { + method.name + } override fun toString(): String = typeName @@ -270,8 +282,17 @@ data class EtsArrayObjectType( } data class EtsUnclearRefType( - override val typeName: String, + val name: String, + val typeParameters: List = emptyList(), ) : EtsRefType { + override val typeName: String + get() = if (typeParameters.isNotEmpty()) { + val generics = typeParameters.joinToString() + "$name<$generics>" + } else { + name + } + override fun toString(): String = typeName override fun accept(visitor: EtsType.Visitor): R { diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 2ba9f7915..5c97bad2d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -625,11 +625,13 @@ fun convertToEtsType(type: TypeDto): EtsType { BooleanTypeDto -> EtsBooleanType is ClassTypeDto -> EtsClassType( - classSignature = convertToEtsClassSignature(type.signature) + signature = convertToEtsClassSignature(type.signature), + typeParameters = type.typeParameters.map { convertToEtsType(it) }, ) is FunctionTypeDto -> EtsFunctionType( - method = convertToEtsMethodSignature(type.signature) + method = convertToEtsMethodSignature(type.signature), + typeParameters = type.typeParameters.map { convertToEtsType(it) }, ) is GenericTypeDto -> { @@ -659,7 +661,8 @@ fun convertToEtsType(type: TypeDto): EtsType { ) is UnclearReferenceTypeDto -> EtsUnclearRefType( - typeName = type.name + name = type.name, + typeParameters = type.typeParameters.map { convertToEtsType(it) }, ) UndefinedTypeDto -> EtsUndefinedType diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt index b8fbf624f..b4a3879f1 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt @@ -65,7 +65,7 @@ data class FieldSignatureDto( val type: TypeDto, ) { override fun toString(): String { - return "$name: $type" + return "${declaringClass.name}::$name: $type" } } @@ -78,7 +78,7 @@ data class MethodSignatureDto( ) { override fun toString(): String { val params = parameters.joinToString() - return "$name($params): $returnType" + return "${declaringClass.name}::$name($params): $returnType" } } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index e110cb42f..df941b905 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -20,6 +20,7 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator +import kotlin.math.sign @Serializable @OptIn(ExperimentalSerializationApi::class) @@ -170,9 +171,15 @@ sealed class PrimitiveLiteralDto { @SerialName("ClassType") data class ClassTypeDto( val signature: ClassSignatureDto, + val typeParameters: List = emptyList(), ) : TypeDto { override fun toString(): String { - return signature.toString() + return if (typeParameters.isNotEmpty()) { + val generics = typeParameters.joinToString() + "$signature<$generics>" + } else { + signature.toString() + } } } @@ -180,9 +187,16 @@ data class ClassTypeDto( @SerialName("FunctionType") data class FunctionTypeDto( val signature: MethodSignatureDto, + val typeParameters: List = emptyList(), ) : TypeDto { override fun toString(): String { - return "(${signature.parameters.joinToString()}) => ${signature.returnType}" + val params = signature.parameters.joinToString() + return if (typeParameters.isNotEmpty()) { + val generics = typeParameters.joinToString() + "${signature.name}<$generics>($params): ${signature.returnType}" + } else { + "${signature.name}($params): ${signature.returnType}" + } } } @@ -201,9 +215,15 @@ data class ArrayTypeDto( @SerialName("UnclearReferenceType") data class UnclearReferenceTypeDto( val name: String, + val typeParameters: List = emptyList(), ) : TypeDto { override fun toString(): String { - return name + return if (typeParameters.isNotEmpty()) { + val generics = typeParameters.joinToString() + "$name<$generics>" + } else { + name + } } } From 1b0e13b263e325d50967b8a3e307c6b017104b4e Mon Sep 17 00:00:00 2001 From: MForest7 Date: Thu, 14 Nov 2024 16:25:13 +0300 Subject: [PATCH 094/120] add resolver stats --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 152 +++++++++++++++++- 1 file changed, 150 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index fb4071bc0..fed2fdddb 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -29,6 +29,7 @@ import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.model.EtsScene import org.jacodb.ets.utils.callExpr import org.jacodb.impl.util.Maybe +import java.util.concurrent.atomic.AtomicInteger private val logger = KotlinLogging.logger {} @@ -89,8 +90,136 @@ class EtsApplicationGraphImpl( private val cacheClassWithIdealSignature: MutableMap> = hashMapOf() private val cacheMethodWithIdealSignature: MutableMap> = hashMapOf() + private val cachePartiallyMatchFailedCallees: MutableSet = hashSetOf() + private val cachePartiallyMultipleMatchesCallees: MutableMap> = hashMapOf() private val cachePartiallyMatchedCallees: MutableMap> = hashMapOf() + data class CalleeStats( + val stmtWithNoCall: AtomicInteger = AtomicInteger(0), + val unknownConstructor: AtomicInteger = AtomicInteger(0), + val constructorOfClassNotFound: AtomicInteger = AtomicInteger(0), + val resolvedConstructor: AtomicInteger = AtomicInteger(0), + val resolvedByTotallyMatchedCache: AtomicInteger = AtomicInteger(0), + val cachedTotalMatchAsUnknown: AtomicInteger = AtomicInteger(0), + val lookupWithIdealSignatureFailed: AtomicInteger = AtomicInteger(0), + val resolvedByIdealSignature: AtomicInteger = AtomicInteger(0), + val resolvedByNeighbour: AtomicInteger = AtomicInteger(0), + val resolvedByPartiallyMatchedCache: AtomicInteger = AtomicInteger(0), + val cachedPartialMatchAsUnknown: AtomicInteger = AtomicInteger(0), + val noPartialMatch: AtomicInteger = AtomicInteger(0), + val multiplePartialMatch: AtomicInteger = AtomicInteger(0), + val partialMatchFound: AtomicInteger = AtomicInteger(0) + ) { + val total: Int + get() = listOf( + unknownConstructor, + constructorOfClassNotFound, + resolvedConstructor, + resolvedByTotallyMatchedCache, + cachedTotalMatchAsUnknown, + lookupWithIdealSignatureFailed, + resolvedByIdealSignature, + resolvedByNeighbour, + resolvedByPartiallyMatchedCache, + cachedPartialMatchAsUnknown, + noPartialMatch, + multiplePartialMatch, + partialMatchFound + ).sumOf { it.get() } + + val resolved: Int + get() = listOf( + resolvedConstructor, + resolvedByNeighbour, + resolvedByIdealSignature, + resolvedByPartiallyMatchedCache, + resolvedByTotallyMatchedCache, + partialMatchFound + ).sumOf { it.get() } + + val unresolved: Int + get() = total - resolved + + val constructors: Int + get() = listOf( + unknownConstructor, + resolvedConstructor, + constructorOfClassNotFound + ).sumOf { it.get() } + + val resolvedByCache: Int + get() = listOf( + resolvedByTotallyMatchedCache, + resolvedByPartiallyMatchedCache + ).sumOf { it.get() } + + val unresolvedByCache: Int + get() = listOf( + cachedTotalMatchAsUnknown, + cachedPartialMatchAsUnknown + ).sumOf { it.get() } + + val cacheHits: Int + get() = resolvedByCache + unresolvedByCache + + val cacheMisses: Int + get() = total - cacheHits + + private fun show(description: String, property: AtomicInteger): String { + if (total == 0) { + return "[N/A%] $description ${property.get()}" + } + val percent = 100 * property.get() / total + return "[$percent%] $description: ${property.get()}" + } + + private fun show(description: String, property: Int): String { + if (total == 0) { + return "[N/A%] $description $property" + } + val percent = 100 * property / total + return "[$percent%] $description: $property" + } + + override fun toString(): String { + return """ + CALLEE RESOLVER STATS: + - Errors: + not a call expression: ${stmtWithNoCall.incrementAndGet()} + - Constructors: + ${show("unknown class signature", unknownConstructor)} + ${show("classpath lookup failed", constructorOfClassNotFound)} + ${show("successfully resolved", resolvedConstructor)} + - Perfect matches: + ${show("resolved by cache", resolvedByTotallyMatchedCache)} + ${show("cached as unknown", cachedTotalMatchAsUnknown)} + ${show("lookup failed", lookupWithIdealSignatureFailed)} + ${show("successfully resolved", resolvedByIdealSignature)} + - Neighbours: + ${show("resolved by neighbour", resolvedByNeighbour)} + - Partial matches: + ${show("resolved by cache", resolvedByPartiallyMatchedCache)} + ${show("cached as unknown", cachedPartialMatchAsUnknown)} + ${show("no partial match", noPartialMatch)} + ${show("multiple partial match", multiplePartialMatch)} + ${show("successfully resolved", partialMatchFound)} + ------------------------------------------------- + Summary: + + ${show("Total", total)} + + + ${show("Resolved", resolved)} + + ${show("Unresolved", unresolved)} + + + ${show("Constructors", constructors)} + + + ${show("Cache hits", cacheHits)} + + ${show("Cache misses", cacheMisses)} + """.trimIndent() + } + } + + val stats = CalleeStats() + private fun lookupClassWithIdealSignature(signature: EtsClassSignature): Maybe { require(signature.isIdeal()) @@ -114,7 +243,10 @@ class EtsApplicationGraphImpl( } override fun callees(node: EtsStmt): Sequence { - val expr = node.callExpr ?: return emptySequence() + val expr = node.callExpr ?: run { + stats.stmtWithNoCall.incrementAndGet() + return emptySequence() + } val callee = expr.method // Note: the resolving code below expects that at least the current method signature is known. @@ -126,6 +258,7 @@ class EtsApplicationGraphImpl( if (callee.name == CONSTRUCTOR_NAME) { if (!callee.enclosingClass.isIdeal()) { // Constructor signature is garbage. Sorry, can't do anything in such case. + stats.unknownConstructor.incrementAndGet() return emptySequence() } @@ -134,8 +267,10 @@ class EtsApplicationGraphImpl( val cls = lookupClassWithIdealSignature(callee.enclosingClass) if (cls.isSome) { + stats.resolvedConstructor.incrementAndGet() return sequenceOf(cls.getOrThrow().ctor) } else { + stats.constructorOfClassNotFound.incrementAndGet() return emptySequence() } } @@ -145,8 +280,10 @@ class EtsApplicationGraphImpl( if (callee in cacheMethodWithIdealSignature) { val resolved = cacheMethodWithIdealSignature.getValue(callee) if (resolved.isSome) { + stats.resolvedByTotallyMatchedCache.incrementAndGet() return sequenceOf(resolved.getOrThrow()) } else { + stats.cachedTotalMatchAsUnknown.incrementAndGet() return emptySequence() } } @@ -162,11 +299,13 @@ class EtsApplicationGraphImpl( } if (resolved.none()) { cacheMethodWithIdealSignature[callee] = Maybe.none() + stats.lookupWithIdealSignatureFailed.incrementAndGet() return emptySequence() } val r = resolved.singleOrNull() ?: error("Multiple methods with the same complete signature: ${resolved.toList()}") cacheMethodWithIdealSignature[callee] = Maybe.some(r) + stats.resolvedByIdealSignature.incrementAndGet() return sequenceOf(r) } @@ -191,12 +330,18 @@ class EtsApplicationGraphImpl( val s = neighbors.singleOrNull() ?: error("Multiple methods with the same name: $neighbors") cachePartiallyMatchedCallees[callee] = listOf(s) + stats.resolvedByNeighbour.incrementAndGet() return sequenceOf(s) } // NOTE: cache lookup MUST be performed AFTER trying to match the neighbour! if (callee in cachePartiallyMatchedCallees) { - return cachePartiallyMatchedCallees.getValue(callee).asSequence() + val s = cachePartiallyMatchedCallees.getValue(callee).asSequence() + if (s.none()) + stats.cachedPartialMatchAsUnknown.incrementAndGet() + else + stats.resolvedByPartiallyMatchedCache.incrementAndGet() + return s } // If the neighbour match failed, @@ -212,14 +357,17 @@ class EtsApplicationGraphImpl( .toList() if (resolved.isEmpty()) { cachePartiallyMatchedCallees[callee] = emptyList() + stats.noPartialMatch.incrementAndGet() return emptySequence() } val r = resolved.singleOrNull() ?: run { logger.warn { "Multiple methods with the same partial signature '${callee}': $resolved" } cachePartiallyMatchedCallees[callee] = emptyList() + stats.multiplePartialMatch.incrementAndGet() return emptySequence() } cachePartiallyMatchedCallees[callee] = listOf(r) + stats.partialMatchFound.incrementAndGet() return sequenceOf(r) } From 4d665efad1b9247403a8602438a9ce9792cee821 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 12:01:09 +0300 Subject: [PATCH 095/120] Query the callees only for call statements --- .../src/main/kotlin/org/jacodb/analysis/ifds/Runner.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Runner.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Runner.kt index 06b7f2ea2..004cdbd04 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Runner.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/ifds/Runner.kt @@ -165,7 +165,6 @@ class UniRunner( val (startVertex, currentVertex) = currentEdge val (current, currentFact) = currentVertex - val currentCallees = graph.callees(current).toList() val currentIsCall = current.callExpr != null val currentIsExit = current in graph.exitPoints(graph.methodOf(current)) @@ -183,7 +182,7 @@ class UniRunner( } // Propagate through the call: - for (callee in currentCallees) { + for (callee in graph.callees(current)) { for (calleeStart in graph.entryPoints(callee)) { val factsAtCalleeStart = flowSpace .obtainCallToStartFlowFunction(current, calleeStart) From 07712933081431405fb7f6407f8b96b2478049a0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 12:15:24 +0300 Subject: [PATCH 096/120] Add kotlin-logging dep for testFixtures --- jacodb-ets/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-ets/build.gradle.kts b/jacodb-ets/build.gradle.kts index 27a60787b..0f6aeca35 100644 --- a/jacodb-ets/build.gradle.kts +++ b/jacodb-ets/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { testImplementation(testFixtures(project(":jacodb-core"))) testImplementation(Libs.mockk) + testFixturesImplementation(Libs.kotlin_logging) testFixturesImplementation(Libs.junit_jupiter_api) } From 91c7779906bd45fa455880f49532e0786dbdd6d0 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 12:18:21 +0300 Subject: [PATCH 097/120] Fix isNullable import --- .../src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt index 42f573f49..505a4b489 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt @@ -58,7 +58,7 @@ import org.jacodb.api.jvm.cfg.JcNewArrayExpr import org.jacodb.api.jvm.cfg.JcNullConstant import org.jacodb.api.jvm.cfg.JcReturnInst import org.jacodb.api.jvm.ext.findType -import org.jacodb.api.jvm.ext.isNullable +import org.jacodb.impl.bytecode.isNullable import org.jacodb.impl.util.onSome import org.jacodb.taint.configuration.AssignMark import org.jacodb.taint.configuration.CopyAllMarks From 5939b32756111754f37117c8be28bed85e38d34e Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 12:18:28 +0300 Subject: [PATCH 098/120] Ignore .kotlin folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e4f1b805b..fcd429928 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .idea/ .gradle/ build/ +.kotlin/ idea-community *.db /generated/ From f9daac51dab0cc8814f20ff5e131b394ed553696 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 12:58:18 +0300 Subject: [PATCH 099/120] Fix tests for FieldInitializers --- .../kotlin/org/jacodb/ets/test/EtsFileTest.kt | 108 ++++++++++++++++-- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt index d3ec69b53..455537b8b 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFileTest.kt @@ -21,6 +21,7 @@ import org.jacodb.ets.base.EtsInstanceFieldRef import org.jacodb.ets.base.EtsLocal import org.jacodb.ets.base.EtsNumberConstant import org.jacodb.ets.base.EtsReturnStmt +import org.jacodb.ets.base.EtsStaticFieldRef import org.jacodb.ets.base.EtsThis import org.jacodb.ets.base.INSTANCE_INIT_METHOD_NAME import org.jacodb.ets.base.STATIC_INIT_METHOD_NAME @@ -87,7 +88,7 @@ class EtsFileTest { val method = cls.methods.single { it.name == INSTANCE_INIT_METHOD_NAME } assertEquals(3, method.cfg.instructions.size) - // Local("this") := ThisRef + // this := ThisRef run { val stmt = method.cfg.instructions[0] assertIs(stmt) @@ -101,7 +102,7 @@ class EtsFileTest { assertEquals("Foo", rhv.type.typeName) } - // Local("this").x := 99 + // this.x := 99 run { val stmt = method.cfg.instructions[1] assertIs(stmt) @@ -134,7 +135,7 @@ class EtsFileTest { val method = cls.methods.single { it.name == STATIC_INIT_METHOD_NAME } assertEquals(3, method.cfg.instructions.size) - // Local("this") := ThisRef + // this := ThisRef run { val stmt = method.cfg.instructions[0] assertIs(stmt) @@ -148,17 +149,16 @@ class EtsFileTest { assertEquals("Foo", rhv.type.typeName) } - // Local("this").y := 111 + // this.y := 111 run { val stmt = method.cfg.instructions[1] assertIs(stmt) val lhv = stmt.lhv - assertIs(lhv) + assertIs(lhv) - val instance = lhv.instance - assertIs(instance) - assertEquals("this", instance.name) + val clazz = lhv.field.enclosingClass + assertEquals("Foo", clazz.name) val field = lhv.field assertEquals("y", field.name) @@ -170,7 +170,97 @@ class EtsFileTest { // return run { - val stmt = method.cfg.instructions[2] + val stmt = method.cfg.instructions.last() + assertIs(stmt) + assertEquals(null, stmt.returnValue) + } + } + + // static field in instance method + run { + val method = cls.methods.single { it.name == "foo" } + + // this := ThisRef + run { + val stmt = method.cfg.instructions[0] + assertIs(stmt) + + val lhv = stmt.lhv + assertIs(lhv) + assertEquals("this", lhv.name) + + val rhv = stmt.rhv + assertIs(rhv) + assertEquals("Foo", rhv.type.typeName) + } + + // Foo.y := 222 + run { + val stmt = method.cfg.instructions[1] + assertIs(stmt) + + val lhv = stmt.lhv + assertIs(lhv) + + val clazz = lhv.field.enclosingClass + assertEquals("Foo", clazz.name) + + val field = lhv.field + assertEquals("y", field.name) + + val rhv = stmt.rhv + assertIs(rhv) + assertEquals(222.0, rhv.value) + } + + // return + run { + val stmt = method.cfg.instructions.last() + assertIs(stmt) + assertEquals(null, stmt.returnValue) + } + } + + // static field in static method + run { + val method = cls.methods.single { it.name == "bar" } + + // this := ThisRef + run { + val stmt = method.cfg.instructions[0] + assertIs(stmt) + + val lhv = stmt.lhv + assertIs(lhv) + assertEquals("this", lhv.name) + + val rhv = stmt.rhv + assertIs(rhv) + assertEquals("Foo", rhv.type.typeName) + } + + // this.y := 333 + run { + val stmt = method.cfg.instructions[1] + assertIs(stmt) + + val lhv = stmt.lhv + assertIs(lhv) + + val clazz = lhv.field.enclosingClass + assertEquals("Foo", clazz.name) + + val field = lhv.field + assertEquals("y", field.name) + + val rhv = stmt.rhv + assertIs(rhv) + assertEquals(333.0, rhv.value) + } + + // return + run { + val stmt = method.cfg.instructions.last() assertIs(stmt) assertEquals(null, stmt.returnValue) } From ac2abfb88dd039b6d435489dbfb064e146cdb29d Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 13:07:50 +0300 Subject: [PATCH 100/120] Migrate to Develocity for Gradle build scans --- settings.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 4d681dd07..24b2ec745 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,14 +1,14 @@ rootProject.name = "jacodb" plugins { - `gradle-enterprise` + id("com.gradle.develocity") version("3.18.2") id("org.danilopianini.gradle-pre-commit-git-hooks") version "1.1.11" } -gradleEnterprise { +develocity { buildScan { - termsOfServiceUrl = "https://gradle.com/terms-of-service" - termsOfServiceAgree = "yes" + termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use") + termsOfUseAgree.set("yes") } } From ba85528475c804cf2ef122feb005deb37cd64373 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 13:10:21 +0300 Subject: [PATCH 101/120] Publish build scans on demand --- settings.gradle.kts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/settings.gradle.kts b/settings.gradle.kts index 24b2ec745..80f8eee8b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -7,8 +7,12 @@ plugins { develocity { buildScan { + // Accept the term of use for the build scan plugin: termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use") termsOfUseAgree.set("yes") + + // Publish build scans on-demand, when `--scan` option is provided: + publishing.onlyIf { false } } } From 58c68f48a93941f25c4d3644176dbe9d4ce8921c Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 14:24:56 +0300 Subject: [PATCH 102/120] Update Gradle wrapper to 6.11 --- gradle/wrapper/gradle-wrapper.jar | Bin 43504 -> 43583 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 2c3521197d7c4586c843d1d3e9090525f1898cde..a4b76b9530d66f5e68d973ea569d8e19de379189 100644 GIT binary patch delta 3990 zcmV;H4{7l5(*nQL0Kr1kzC=_KMxQY0|W5(lc#i zH*M1^P4B}|{x<+fkObwl)u#`$GxKKV&3pg*-y6R6txw)0qU|Clf9Uds3x{_-**c=7 z&*)~RHPM>Rw#Hi1R({;bX|7?J@w}DMF>dQQU2}9yj%iLjJ*KD6IEB2^n#gK7M~}6R zkH+)bc--JU^pV~7W=3{E*4|ZFpDpBa7;wh4_%;?XM-5ZgZNnVJ=vm!%a2CdQb?oTa z70>8rTb~M$5Tp!Se+4_OKWOB1LF+7gv~$$fGC95ToUM(I>vrd$>9|@h=O?eARj0MH zT4zo(M>`LWoYvE>pXvqG=d96D-4?VySz~=tPVNyD$XMshoTX(1ZLB5OU!I2OI{kb) zS8$B8Qm>wLT6diNnyJZC?yp{Kn67S{TCOt-!OonOK7$K)e-13U9GlnQXPAb&SJ0#3 z+vs~+4Qovv(%i8g$I#FCpCG^C4DdyQw3phJ(f#y*pvNDQCRZ~MvW<}fUs~PL=4??j zmhPyg<*I4RbTz|NHFE-DC7lf2=}-sGkE5e!RM%3ohM7_I^IF=?O{m*uUPH(V?gqyc(Rp?-Qu(3bBIL4Fz(v?=_Sh?LbK{nqZMD>#9D_hNhaV$0ef3@9V90|0u#|PUNTO>$F=qRhg1duaE z0`v~X3G{8RVT@kOa-pU+z8{JWyP6GF*u2e8eKr7a2t1fuqQy)@d|Qn(%YLZ62TWtoX@$nL}9?atE#Yw`rd(>cr0gY;dT9~^oL;u)zgHUvxc2I*b&ZkGM-iq=&(?kyO(3}=P! zRp=rErEyMT5UE9GjPHZ#T<`cnD)jyIL!8P{H@IU#`e8cAG5jMK zVyKw7--dAC;?-qEu*rMr$5@y535qZ6p(R#+fLA_)G~!wnT~~)|s`}&fA(s6xXN`9j zP#Fd3GBa#HeS{5&8p?%DKUyN^X9cYUc6vq}D_3xJ&d@=6j(6BZKPl?!k1?!`f3z&a zR4ZF60Mx7oBxLSxGuzA*Dy5n-d2K=+)6VMZh_0KetK|{e;E{8NJJ!)=_E~1uu=A=r zrn&gh)h*SFhsQJo!f+wKMIE;-EOaMSMB@aXRU(UcnJhZW^B^mgs|M9@5WF@s6B0p& zm#CTz)yiQCgURE{%hjxHcJ6G&>G9i`7MyftL!QQd5 z@RflRs?7)99?X`kHNt>W3l7YqscBpi*R2+fsgABor>KVOu(i(`03aytf2UA!&SC9v z!E}whj#^9~=XHMinFZ;6UOJjo=mmNaWkv~nC=qH9$s-8roGeyaW-E~SzZ3Gg>j zZ8}<320rg4=$`M0nxN!w(PtHUjeeU?MvYgWKZ6kkzABK;vMN0|U;X9abJleJA(xy<}5h5P(5 z{RzAFPvMnX2m0yH0Jn2Uo-p`daE|(O`YQiC#jB8;6bVIUf?SY(k$#C0`d6qT`>Xe0+0}Oj0=F&*D;PVe=Z<=0AGI<6$gYLwa#r` zm449x*fU;_+J>Mz!wa;T-wldoBB%&OEMJgtm#oaI60TSYCy7;+$5?q!zi5K`u66Wq zvg)Fx$s`V3Em{=OEY{3lmh_7|08ykS&U9w!kp@Ctuzqe1JFOGz6%i5}Kmm9>^=gih z?kRxqLA<3@e=}G4R_?phW{4DVr?`tPfyZSN@R=^;P;?!2bh~F1I|fB7P=V=9a6XU5 z<#0f>RS0O&rhc&nTRFOW7&QhevP0#>j0eq<1@D5yAlgMl5n&O9X|Vq}%RX}iNyRFF z7sX&u#6?E~bm~N|z&YikXC=I0E*8Z$v7PtWfjy)$e_Ez25fnR1Q=q1`;U!~U>|&YS zaOS8y!^ORmr2L4ik!IYR8@Dcx8MTC=(b4P6iE5CnrbI~7j7DmM8em$!da&D!6Xu)!vKPdLG z9f#)se|6=5yOCe)N6xDhPI!m81*dNe7u985zi%IVfOfJh69+#ag4ELzGne?o`eA`42K4T)h3S+s)5IT97%O>du- z0U54L8m4}rkRQ?QBfJ%DLssy^+a7Ajw;0&`NOTY4o;0-ivm9 zBz1C%nr_hQ)X)^QM6T1?=yeLkuG9Lf50(eH}`tFye;01&(p?8i+6h};VV-2B~qdxeC#=X z(JLlzy&fHkyi9Ksbcs~&r^%lh^2COldLz^H@X!s~mr9Dr6z!j+4?zkD@Ls7F8(t(f z9`U?P$Lmn*Y{K}aR4N&1N=?xtQ1%jqf1~pJyQ4SgBrEtR`j4lQuh7cqP49Em5cO=I zB(He2`iPN5M=Y0}h(IU$37ANTGx&|b-u1BYA*#dE(L-lptoOpo&th~E)_)y-`6kSH z3vvyVrcBwW^_XYReJ=JYd9OBQrzv;f2AQdZH#$Y{Y+Oa33M70XFI((fs;mB4e`<<{ ze4dv2B0V_?Ytsi>>g%qs*}oDGd5d(RNZ*6?7qNbdp7wP4T72=F&r?Ud#kZr8Ze5tB z_oNb7{G+(o2ajL$!69FW@jjPQ2a5C)m!MKKRirC$_VYIuVQCpf9rIms0GRDf)8AH${I`q^~5rjot@#3$2#zT2f`(N^P7Z;6(@EK$q*Jgif00I6*^ZGV+XB5uw*1R-@23yTw&WKD{s1;HTL;dO)%5i#`dc6b7;5@^{KU%N|A-$zsYw4)7LA{3`Zp>1 z-?K9_IE&z)dayUM)wd8K^29m-l$lFhi$zj0l!u~4;VGR6Y!?MAfBC^?QD53hy6VdD z@eUZIui}~L%#SmajaRq1J|#> z4m=o$vZ*34=ZWK2!QMNEcp2Lbc5N1q!lEDq(bz0b;WI9;e>l=CG9^n#ro`w>_0F$Q zfZ={2QyTkfByC&gy;x!r*NyXXbk=a%~~(#K?< zTke0HuF5{Q+~?@!KDXR|g+43$+;ab`^flS%miup_0OUTm=nIc%d5nLP)i308PIjl_YMF6cpQ__6&$n6it8K- z8PIjl_YMF6cpQ_!r)L8IivW`WdK8mBs6PXdjR2DYdK8nCs73=4j{uVadK8oNjwX|E wpAeHLsTu^*Y>Trk?aBtSQ(D-o$(D8Px^?ZI-PUB? z*1fv!{YdHme3Fc8%cR@*@zc5A_nq&2=R47Hp@$-JF4Fz*;SLw5}K^y>s-s;V!}b2i=5=M- zComP?ju>8Fe@=H@rlwe1l`J*6BTTo`9b$zjQ@HxrAhp0D#u?M~TxGC_!?ccCHCjt| zF*PgJf@kJB`|Ml}cmsyrAjO#Kjr^E5p29w+#>$C`Q|54BoDv$fQ9D?3n32P9LPMIzu?LjNqggOH=1@T{9bMn*u8(GI z!;MLTtFPHal^S>VcJdiYqX0VU|Rn@A}C1xOlxCribxes0~+n2 z6qDaIA2$?e`opx3_KW!rAgbpzU)gFdjAKXh|5w``#F0R|c)Y)Du0_Ihhz^S?k^pk% zP>9|pIDx)xHH^_~+aA=^$M!<8K~Hy(71nJGf6`HnjtS=4X4=Hk^O71oNia2V{HUCC zoN3RSBS?mZCLw;l4W4a+D8qc)XJS`pUJ5X-f^1ytxwr`@si$lAE?{4G|o; zO0l>`rr?;~c;{ZEFJ!!3=7=FdGJ?Q^xfNQh4A?i;IJ4}B+A?4olTK(fN++3CRBP97 ze~lG9h%oegkn)lpW-4F8o2`*WW0mZHwHez`ko@>U1_;EC_6ig|Drn@=DMV9YEUSCa zIf$kHei3(u#zm9I!Jf(4t`Vm1lltJ&lVHy(eIXE8sy9sUpmz%I_gA#8x^Zv8%w?r2 z{GdkX1SkzRIr>prRK@rqn9j2wG|rUvf6PJbbin=yy-TAXrguvzN8jL$hUrIXzr^s5 zVM?H4;eM-QeRFr06@ifV(ocvk?_)~N@1c2ien56UjWXid6W%6ievIh)>dk|rIs##^kY67ib8Kw%#-oVFaXG7$ERyA9(NSJUvWiOA5H(!{uOpcW zg&-?iqPhds%3%tFspHDqqr;A!e@B#iPQjHd=c>N1LoOEGRehVoPOdxJ>b6>yc#o#+ zl8s8!(|NMeqjsy@0x{8^j0d00SqRZjp{Kj)&4UHYGxG+z9b-)72I*&J70?+8e?p_@ z=>-(>l6z5vYlP~<2%DU02b!mA{7mS)NS_eLe=t)sm&+Pmk?asOEKlkPQ)EUvvfC=;4M&*|I!w}(@V_)eUKLA_t^%`o z0PM9LV|UKTLnk|?M3u!|f2S0?UqZsEIH9*NJS-8lzu;A6-rr-ot=dg9SASoluZUkFH$7X; zP=?kYX!K?JL-b~<#7wU;b;eS)O;@?h%sPPk{4xEBxb{!sm0AY|f9cNvx6>$3F!*0c z75H=dy8JvTyO8}g1w{$9T$p~5en}AeSLoCF>_RT9YPMpChUjl310o*$QocjbH& zbnwg#gssR#jDVN{uEi3n(PZ%PFZ|6J2 z5_rBf0-u>e4sFe0*Km49ATi7>Kn0f9!uc|rRMR1Dtt6m1LW8^>qFlo}h$@br=Rmpi z;mI&>OF64Be{dVeHI8utrh)v^wsZ0jii%x8UgZ8TC%K~@I(4E};GFW&(;WVov}3%H zH;IhRkfD^(vt^DjZz(MyHLZxv8}qzPc(%itBkBwf_fC~sDBgh<3XAv5cxxfF3<2U! z03Xe&z`is!JDHbe;mNmfkH+_LFE*I2^mdL@7(@9DfAcP6O04V-ko;Rpgp<%Cj5r8Z zd0`sXoIjV$j)--;jA6Zy^D5&5v$o^>e%>Q?9GLm{i~p^lAn!%ZtF$I~>39XVZxk0b zROh^Bk9cE0AJBLozZIEmy7xG(yHWGztvfnr0(2ro1%>zsGMS^EMu+S$r=_;9 zWwZkgf7Q7`H9sLf2Go^Xy6&h~a&%s2_T@_Csf19MntF$aVFiFkvE3_hUg(B@&Xw@YJ zpL$wNYf78=0c@!QU6_a$>CPiXT7QAGDM}7Z(0z#_ZA=fmLUj{2z7@Ypo71UDy8GHr z-&TLKf6a5WCf@Adle3VglBt4>Z>;xF}}-S~B7<(%B;Y z0QR55{z-buw>8ilNM3u6I+D$S%?)(p>=eBx-HpvZj{7c*_?K=d()*7q?93us}1dq%FAFYLsW8ZTQ_XZLh`P2*6(NgS}qGcfGXVWpwsp#Rs}IuKbk*`2}&) zI^Vsk6S&Q4@oYS?dJ`NwMVBs6f57+RxdqVub#PvMu?$=^OJy5xEl0<5SLsSRy%%a0 zi}Y#1-F3m;Ieh#Y12UgW?-R)|eX>ZuF-2cc!1>~NS|XSF-6In>zBoZg+ml!6%fk7U zw0LHcz8VQk(jOJ+Yu)|^|15ufl$KQd_1eUZZzj`aC%umU6F1&D5XVWce_wAe(qCSZ zpX-QF4e{EmEVN9~6%bR5U*UT{eMHfcUo`jw*u?4r2s_$`}U{?NjvEm(u&<>B|%mq$Q3weshxk z76<``8vh{+nX`@9CB6IE&z)I%IFjR^LH{s1p|eppv=x za(g_jLU|xjWMAn-V7th$f({|LG8zzIE0g?cyW;%Dmtv%C+0@xVxPE^ zyZzi9P%JAD6ynwHptuzP`Kox7*9h7XSMonCalv;Md0i9Vb-c*!f0ubfk?&T&T}AHh z4m8Bz{JllKcdNg?D^%a5MFQ;#1z|*}H^qHLzW)L}wp?2tY7RejtSh8<;Zw)QGJYUm z|MbTxyj*McKlStlT9I5XlSWtQGN&-LTr2XyNU+`490rg?LYLMRnz-@oKqT1hpCGqP zyRXt4=_Woj$%n5ee<3zhLF>5>`?m9a#xQH+Jk_+|RM8Vi;2*XbK- zEL6sCpaGPzP>k8f4Kh|##_imt#zJMB;ir|JrMPGW`rityK1vHXMLy18%qmMQAm4WZ zP)i30KR&5vs15)C+8dM66&$k~i|ZT;KR&5vs15)C+8dJ(sAmGPijyIz6_bsqKLSFH zlOd=TljEpH0>h4zA*dCTK&emy#FCRCs1=i^sZ9bFmXjf<6_X39E(XY)00000#N437 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index dedd5d1e6..7cf748e74 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From aa1b8a8fdfef6d7e2f7af07f906e06a882927f4a Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 14:27:59 +0300 Subject: [PATCH 103/120] Remove context receivers on classes Context-in-class feature was dropped with a new KEEP-367 "Context parameters" (see https://github.com/Kotlin/KEEP/blob/context-parameters/proposals/context-parameters.md), which is going to replace "Context receivers" from KEEP-259. Context declarations on classes will become an error in Kotlin 2.1, so we are preparing to move away from them in advance. --- .../org/jacodb/analysis/config/Condition.kt | 20 ++--- .../org/jacodb/analysis/config/Position.kt | 29 ++++--- .../org/jacodb/analysis/npe/NpeAnalyzers.kt | 13 +-- .../jacodb/analysis/npe/NpeFlowFunctions.kt | 79 +++++++++-------- .../org/jacodb/analysis/npe/NpeManager.kt | 6 +- .../jacodb/analysis/taint/TaintAnalyzers.kt | 23 ++--- .../analysis/taint/TaintFlowFunctions.kt | 86 ++++++++++--------- .../org/jacodb/analysis/taint/TaintManager.kt | 10 +-- .../analysis/unused/UnusedVariableAnalyzer.kt | 4 +- .../unused/UnusedVariableFlowFunctions.kt | 15 ++-- .../analysis/unused/UnusedVariableManager.kt | 6 +- .../org/jacodb/analysis/util/JcTraits.kt | 19 +--- .../analysis/impl/JavaAnalysisApiTest.java | 3 +- .../jacodb/analysis/impl/BaseAnalysisTest.kt | 8 +- .../analysis/impl/ConditionEvaluatorTest.kt | 9 +- .../org/jacodb/analysis/impl/IfdsNpeTest.kt | 4 +- .../org/jacodb/analysis/impl/IfdsSqlTest.kt | 8 +- .../org/jacodb/analysis/impl/IfdsTaintTest.kt | 2 +- .../impl/IfdsUntrustedLoopBoundTest.kt | 5 +- .../jacodb/analysis/impl/IfdsUnusedTest.kt | 4 +- .../analysis/impl/JodaDateTimeAnalysisTest.kt | 9 +- .../analysis/impl/TaintFlowFunctionsTest.kt | 54 ++++++------ .../src/main/kotlin/org/jacodb/cli/main.kt | 13 +-- .../kotlin/org/jacodb/ets/test/EtsIfds.kt | 11 ++- .../org/jacodb/ets/test/EtsProjectAnalysis.kt | 11 ++- .../jacodb/ets/test/EtsTaintAnalysisTest.kt | 5 +- .../src/test/kotlin/panda/PandaDemoCases.kt | 8 +- .../src/test/kotlin/panda/PandaIfdsTest.kt | 5 +- 28 files changed, 246 insertions(+), 223 deletions(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt index 376fae3db..d9cc58377 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt @@ -45,8 +45,8 @@ import org.jacodb.taint.configuration.TypeMatches // TODO: replace 'JcInt' with 'CommonInt', etc -context(Traits) open class BasicConditionEvaluator( + val traits: Traits, internal val positionResolver: PositionResolver>, ) : ConditionVisitor { @@ -78,35 +78,35 @@ open class BasicConditionEvaluator( error("Unexpected condition: $condition") } - override fun visit(condition: IsConstant): Boolean { + override fun visit(condition: IsConstant): Boolean = with(traits) { positionResolver.resolve(condition.position).onSome { return it.isConstant() } return false } - override fun visit(condition: ConstantEq): Boolean { + override fun visit(condition: ConstantEq): Boolean = with(traits) { positionResolver.resolve(condition.position).onSome { value -> return value.eqConstant(condition.value) } return false } - override fun visit(condition: ConstantLt): Boolean { + override fun visit(condition: ConstantLt): Boolean = with(traits) { positionResolver.resolve(condition.position).onSome { value -> return value.ltConstant(condition.value) } return false } - override fun visit(condition: ConstantGt): Boolean { + override fun visit(condition: ConstantGt): Boolean = with(traits) { positionResolver.resolve(condition.position).onSome { value -> return value.gtConstant(condition.value) } return false } - override fun visit(condition: ConstantMatches): Boolean { + override fun visit(condition: ConstantMatches): Boolean = with(traits){ positionResolver.resolve(condition.position).onSome { value -> return value.matches(condition.pattern) } @@ -135,13 +135,13 @@ open class BasicConditionEvaluator( } } -context(Traits) class FactAwareConditionEvaluator( - private val fact: Tainted, + traits: Traits, positionResolver: PositionResolver>, -) : BasicConditionEvaluator(positionResolver) { + private val fact: Tainted, +) : BasicConditionEvaluator(traits, positionResolver) { - override fun visit(condition: ContainsMark): Boolean { + override fun visit(condition: ContainsMark): Boolean = with(traits) { if (fact.mark != condition.mark) return false positionResolver.resolve(condition.position).onSome { value -> val variable = value.toPath() diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt index 347e72497..6fc74c68b 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt @@ -35,14 +35,16 @@ import org.jacodb.taint.configuration.Result import org.jacodb.taint.configuration.ResultAnyElement import org.jacodb.taint.configuration.This -context(Traits) class CallPositionToAccessPathResolver( + val traits: Traits, private val callStatement: CommonInst, ) : PositionResolver> { - private val callExpr = callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") + private val callExpr = with(traits) { + callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") + } - override fun resolve(position: Position): Maybe = when (position) { + override fun resolve(position: Position): Maybe = with(traits) {when (position) { AnyArgument -> Maybe.none() is Argument -> callExpr.args[position.index].toPathOrNull().toMaybe() This -> (callExpr as? CommonInstanceCallExpr)?.instance?.toPathOrNull().toMaybe() @@ -50,14 +52,17 @@ class CallPositionToAccessPathResolver( ResultAnyElement -> (callStatement as? CommonAssignInst)?.lhv?.toPathOrNull().toMaybe() .fmap { it + ElementAccessor } } + } } -context(Traits) class CallPositionToValueResolver( + val traits: Traits, private val callStatement: CommonInst, ) : PositionResolver> { - private val callExpr = callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") + private val callExpr = with(traits) { + callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") + } override fun resolve(position: Position): Maybe = when (position) { AnyArgument -> Maybe.none() @@ -68,11 +73,11 @@ class CallPositionToValueResolver( } } -context(Traits) class EntryPointPositionToValueResolver( + val traits: Traits, private val method: CommonMethod, ) : PositionResolver> { - override fun resolve(position: Position): Maybe = when (position) { + override fun resolve(position: Position): Maybe = with(traits) {when (position) { This -> Maybe.some(method.thisInstance) is Argument -> { @@ -82,13 +87,14 @@ class EntryPointPositionToValueResolver( AnyArgument, Result, ResultAnyElement -> error("Unexpected $position") } + } } -context(Traits) class EntryPointPositionToAccessPathResolver( + val traits: Traits, private val method: CommonMethod, ) : PositionResolver> { - override fun resolve(position: Position): Maybe = when (position) { + override fun resolve(position: Position): Maybe = with(traits){ when (position) { This -> method.thisInstance.toPathOrNull().toMaybe() is Argument -> { @@ -98,4 +104,5 @@ class EntryPointPositionToAccessPathResolver( AnyArgument, Result, ResultAnyElement -> error("Unexpected $position") } + } } diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt index aef9e63a7..4c87032c3 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt @@ -40,15 +40,15 @@ import org.jacodb.taint.configuration.TaintMethodSink private val logger = mu.KotlinLogging.logger {} -context(Traits) class NpeAnalyzer( + private val traits: Traits, private val graph: ApplicationGraph, ) : Analyzer, Method, Statement> where Method : CommonMethod, Statement : CommonInst { override val flowFunctions: ForwardNpeFlowFunctions by lazy { - ForwardNpeFlowFunctions(graph) + ForwardNpeFlowFunctions(traits, graph) } private val taintConfigurationFeature: TaintConfigurationFeature? @@ -66,7 +66,7 @@ class NpeAnalyzer( } if (edge.to.fact is Tainted && edge.to.fact.mark == TaintMark.NULLNESS) { - if (edge.to.fact.variable.isDereferencedAt(edge.to.statement)) { + if (with(traits) { edge.to.fact.variable.isDereferencedAt(edge.to.statement) }) { val message = "NPE" // TODO val vulnerability = TaintVulnerability(message, sink = edge.to) logger.info { @@ -78,8 +78,8 @@ class NpeAnalyzer( } run { - val callExpr = edge.to.statement.getCallExpr() ?: return@run - val callee = callExpr.callee + val callExpr = with(traits) { edge.to.statement.getCallExpr() } ?: return@run + val callee = with(traits) { callExpr.callee } val config = taintConfigurationFeature?.let { feature -> if (callee is JcMethod) { @@ -98,8 +98,9 @@ class NpeAnalyzer( // Determine whether 'edge.to' is a sink via config: val conditionEvaluator = FactAwareConditionEvaluator( + traits, + CallPositionToValueResolver(traits, edge.to.statement), edge.to.fact, - CallPositionToValueResolver(edge.to.statement), ) for (item in config.filterIsInstance()) { if (item.condition.accept(conditionEvaluator)) { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt index 505a4b489..5b9346e93 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt @@ -74,8 +74,8 @@ import org.jacodb.taint.configuration.TaintPassThrough private val logger = mu.KotlinLogging.logger {} -context(Traits) class ForwardNpeFlowFunctions( + private val traits: Traits, private val graph: ApplicationGraph, ) : FlowFunctions where Method : CommonMethod, @@ -103,7 +103,7 @@ class ForwardNpeFlowFunctions( for (p in method.parameters.filter { it.isNullable != false }) { val t = (graph as JcApplicationGraph).cp.findType(p.type.typeName) val arg = JcArgument.of(p.index, p.name, t) - val path = arg.toPath() + val path = with(traits) { arg.toPath() } add(Tainted(path, TaintMark.NULLNESS)) } } @@ -124,8 +124,8 @@ class ForwardNpeFlowFunctions( } } if (config != null) { - val conditionEvaluator = BasicConditionEvaluator(EntryPointPositionToValueResolver(method)) - val actionEvaluator = TaintActionEvaluator(EntryPointPositionToAccessPathResolver(method)) + val conditionEvaluator = BasicConditionEvaluator(traits, EntryPointPositionToValueResolver(traits, method)) + val actionEvaluator = TaintActionEvaluator(EntryPointPositionToAccessPathResolver(traits,method)) // Handle EntryPointSource config items: for (item in config.filterIsInstance()) { @@ -147,8 +147,8 @@ class ForwardNpeFlowFunctions( from: JcExpr, to: CommonValue, ): Collection { - val toPath = to.toPath() - val fromPath = from.toPathOrNull() + val toPath = with(traits){ to.toPath() } + val fromPath = with(traits) { from.toPathOrNull() } if (fact.mark == TaintMark.NULLNESS) { // TODO: consider @@ -208,7 +208,7 @@ class ForwardNpeFlowFunctions( inst: Statement, ): Collection = buildList { if (inst is CommonAssignInst) { - val toPath = inst.lhv.toPath() + val toPath = with(traits) { inst.lhv.toPath() } val from = inst.rhv if (from is JcNullConstant || (from is JcCallExpr && from.method.method.isNullable == true)) { add(Tainted(toPath, TaintMark.NULLNESS)) @@ -224,9 +224,9 @@ class ForwardNpeFlowFunctions( get() { val expr = condition return if (expr.rhv is JcNullConstant) { - expr.lhv.toPathOrNull() + with(traits) { expr.lhv.toPathOrNull() } } else if (expr.lhv is JcNullConstant) { - expr.rhv.toPathOrNull() + with(traits) { expr.rhv.toPathOrNull() } } else { null } @@ -237,7 +237,7 @@ class ForwardNpeFlowFunctions( next: Statement, ) = FlowFunction { fact -> if (fact is Tainted && fact.mark == TaintMark.NULLNESS) { - if (fact.variable.isDereferencedAt(current)) { + if ( with(traits){ fact.variable.isDereferencedAt(current) }) { return@FlowFunction emptySet() } } @@ -290,13 +290,13 @@ class ForwardNpeFlowFunctions( to: CommonValue, ): Collection = buildSet { if (fact.mark == TaintMark.NULLNESS) { - if (fact.variable.isDereferencedAt(at)) { + if ( with(traits){ fact.variable.isDereferencedAt(at) }) { return@buildSet } } - val fromPath = from.toPath() - val toPath = to.toPath() + val fromPath = with(traits){ from.toPath() } + val toPath = with(traits){ to.toPath() } val tail = (fact.variable - fromPath) ?: return@buildSet val newPath = toPath + tail @@ -344,15 +344,17 @@ class ForwardNpeFlowFunctions( returnSite: Statement, // FIXME: unused? ) = FlowFunction { fact -> if (fact is Tainted && fact.mark == TaintMark.NULLNESS) { - if (fact.variable.isDereferencedAt(callStatement)) { + if ( with(traits){ fact.variable.isDereferencedAt(callStatement) }) { return@FlowFunction emptySet() } } - val callExpr = callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") + val callExpr = with(traits) { + callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") + } - val callee = callExpr.callee + val callee = with(traits){ callExpr.callee } // FIXME: handle taint pass-through on invokedynamic-based String concatenation: if (fact is Tainted @@ -361,10 +363,10 @@ class ForwardNpeFlowFunctions( && callStatement is JcAssignInst ) { for (arg in callExpr.args) { - if (arg.toPath() == fact.variable) { + if ( with(traits) { arg.toPath() } == fact.variable) { return@FlowFunction setOf( fact, - fact.copy(variable = callStatement.lhv.toPath()) + fact.copy(variable = with(traits){callStatement.lhv.toPath()}) ) } } @@ -385,7 +387,7 @@ class ForwardNpeFlowFunctions( add(TaintZeroFact) if (callStatement is JcAssignInst) { - val toPath = callStatement.lhv.toPath() + val toPath = with(traits){ callStatement.lhv.toPath() } val from = callStatement.rhv if (from is JcNullConstant || (from is JcCallExpr && from.method.method.isNullable == true)) { add(Tainted(toPath, TaintMark.NULLNESS)) @@ -399,10 +401,11 @@ class ForwardNpeFlowFunctions( if (config != null) { val conditionEvaluator = BasicConditionEvaluator( - CallPositionToValueResolver(callStatement) + traits, + CallPositionToValueResolver(traits, callStatement) ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(callStatement) + CallPositionToAccessPathResolver(traits,callStatement) ) // Handle MethodSource config items: @@ -434,10 +437,12 @@ class ForwardNpeFlowFunctions( } else { val facts = mutableSetOf() val conditionEvaluator = FactAwareConditionEvaluator( - fact, CallPositionToValueResolver(callStatement) + traits, + CallPositionToValueResolver(traits,callStatement), + fact ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(callStatement) + CallPositionToAccessPathResolver(traits, callStatement) ) var defaultBehavior = true @@ -489,7 +494,7 @@ class ForwardNpeFlowFunctions( } // FIXME: adhoc for constructors: - if (callee.isConstructor) { + if ( with(traits) { callee.isConstructor }) { return@FlowFunction listOf(fact) } @@ -510,14 +515,14 @@ class ForwardNpeFlowFunctions( for (actual in callExpr.args) { // Possibly tainted actual parameter: - if (fact.variable.startsWith(actual.toPathOrNull())) { + if (fact.variable.startsWith( with(traits){ actual.toPathOrNull() })) { return@FlowFunction emptyList() // Will be handled by summary edge } } if (callExpr is JcInstanceCallExpr) { // Possibly tainted instance: - if (fact.variable.startsWith(callExpr.instance.toPathOrNull())) { + if (fact.variable.startsWith( with(traits){ callExpr.instance.toPathOrNull() })) { return@FlowFunction emptyList() // Will be handled by summary edge } } @@ -526,7 +531,7 @@ class ForwardNpeFlowFunctions( if (callStatement is JcAssignInst) { // Possibly tainted lhv: - if (fact.variable.startsWith(callStatement.lhv.toPathOrNull())) { + if (fact.variable.startsWith( with(traits){ callStatement.lhv.toPathOrNull() })) { return@FlowFunction emptyList() // Overridden by rhv } } @@ -546,13 +551,15 @@ class ForwardNpeFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") + val callExpr = with(traits) { + callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") + } buildSet { // Transmit facts on arguments (from 'actual' to 'formal'): val actualParams = callExpr.args - val formalParams = getArgumentsOf(callee) + val formalParams = with(traits){ getArgumentsOf(callee) } for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentActualToFormal( @@ -571,7 +578,7 @@ class ForwardNpeFlowFunctions( fact = fact, at = callStatement, from = callExpr.instance, - to = callee.thisInstance + to = with(traits) { callee.thisInstance } ) ) } @@ -597,7 +604,7 @@ class ForwardNpeFlowFunctions( // Note: returnValue can be null here in some weird cases, e.g. in lambda. exitStatement.returnValue?.let { returnValue -> if (returnValue is JcNullConstant) { - val toPath = callStatement.lhv.toPath() + val toPath = with(traits) { callStatement.lhv.toPath() } add(Tainted(toPath, TaintMark.NULLNESS)) } } @@ -606,7 +613,7 @@ class ForwardNpeFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits){ callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") val callee = graph.methodOf(exitStatement) @@ -614,7 +621,7 @@ class ForwardNpeFlowFunctions( // Transmit facts on arguments (from 'formal' back to 'actual'), if they are passed by-ref: if (fact.variable.isOnHeap) { val actualParams = callExpr.args - val formalParams = getArgumentsOf(callee) + val formalParams = with(traits) { getArgumentsOf(callee) } for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentFormalToActual( @@ -633,7 +640,7 @@ class ForwardNpeFlowFunctions( transmitTaintThisToInstance( fact = fact, at = callStatement, - from = callee.thisInstance, + from = with(traits){ callee.thisInstance }, to = callExpr.instance ) ) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt index a9e52776d..337ec1ec6 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt @@ -30,11 +30,11 @@ import org.jacodb.api.common.cfg.CommonInst private val logger = mu.KotlinLogging.logger {} -context(Traits) class NpeManager( + traits: Traits, graph: ApplicationGraph, unitResolver: UnitResolver, -) : TaintManager(graph, unitResolver, useBidiRunner = false) +) : TaintManager(traits, graph, unitResolver, useBidiRunner = false) where Method : CommonMethod, Statement : CommonInst { @@ -43,7 +43,7 @@ class NpeManager( ): TaintRunner { check(unit !in runnerForUnit) { "Runner for $unit already exists" } - val analyzer = NpeAnalyzer(graph) + val analyzer = NpeAnalyzer(traits, graph) val runner = UniRunner( graph = graph, analyzer = analyzer, diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt index 675c9c2a1..28f634d6a 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt @@ -27,7 +27,6 @@ import org.jacodb.api.common.CommonMethod import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.api.common.cfg.CommonInst import org.jacodb.api.jvm.cfg.JcIfInst -import org.jacodb.impl.cfg.util.loops import org.jacodb.ets.base.EtsArrayAccess import org.jacodb.ets.base.EtsAssignStmt import org.jacodb.ets.base.EtsIfStmt @@ -35,6 +34,7 @@ import org.jacodb.ets.base.EtsNewArrayExpr import org.jacodb.ets.base.EtsStmt import org.jacodb.ets.graph.loops import org.jacodb.ets.utils.getOperands +import org.jacodb.impl.cfg.util.loops import org.jacodb.panda.staticvm.utils.loops import org.jacodb.taint.configuration.TaintConfigurationItem import org.jacodb.taint.configuration.TaintMethodSink @@ -42,8 +42,8 @@ import org.jacodb.panda.staticvm.cfg.PandaIfInst as StaticPandaIfInst private val logger = mu.KotlinLogging.logger {} -context(Traits) class TaintAnalyzer( + val traits: Traits, private val graph: ApplicationGraph, private val getConfigForMethod: (ForwardTaintFlowFunctions.(Method) -> List?)? = null, ) : Analyzer, Method, Statement> @@ -52,9 +52,9 @@ class TaintAnalyzer( override val flowFunctions: ForwardTaintFlowFunctions by lazy { if (getConfigForMethod != null) { - ForwardTaintFlowFunctions(graph, getConfigForMethod) + ForwardTaintFlowFunctions(traits, graph, getConfigForMethod) } else { - ForwardTaintFlowFunctions(graph) + ForwardTaintFlowFunctions(traits, graph) } } @@ -70,9 +70,9 @@ class TaintAnalyzer( } run { - val callExpr = edge.to.statement.getCallExpr() ?: return@run + val callExpr = with(traits) { edge.to.statement.getCallExpr() } ?: return@run - val callee = callExpr.callee + val callee = with(traits) { callExpr.callee } val config = with(flowFunctions) { getConfigForMethod(callee) } ?: return@run @@ -84,8 +84,9 @@ class TaintAnalyzer( // Determine whether 'edge.to' is a sink via config: val conditionEvaluator = FactAwareConditionEvaluator( + traits, + CallPositionToValueResolver(traits, edge.to.statement), edge.to.fact, - CallPositionToValueResolver(edge.to.statement), ) for (item in config.filterIsInstance()) { if (item.condition.accept(conditionEvaluator)) { @@ -152,7 +153,7 @@ class TaintAnalyzer( val expr = statement.rhv if (expr is EtsNewArrayExpr) { val arg = expr.size - if (arg.toPathOrNull() == fact.variable) { + if (with(traits) { arg.toPathOrNull() } == fact.variable) { val message = "Untrusted array size" val vulnerability = TaintVulnerability(message, sink = edge.to) add(NewVulnerability(vulnerability)) @@ -169,7 +170,7 @@ class TaintAnalyzer( for (op in statement.getOperands()) { if (op is EtsArrayAccess) { val arg = op.index - if (arg.toPathOrNull() == fact.variable) { + if (with(traits) { arg.toPathOrNull() } == fact.variable) { val message = "Untrusted index for access array" val vulnerability = TaintVulnerability(message, sink = edge.to) add(NewVulnerability(vulnerability)) @@ -189,15 +190,15 @@ class TaintAnalyzer( } } -context(Traits) class BackwardTaintAnalyzer( + val traits: Traits, private val graph: ApplicationGraph, ) : Analyzer, Method, Statement> where Method : CommonMethod, Statement : CommonInst { override val flowFunctions: BackwardTaintFlowFunctions by lazy { - BackwardTaintFlowFunctions(graph) + BackwardTaintFlowFunctions(traits, graph) } private fun isExitPoint(statement: Statement): Boolean { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt index 6a4cf1abf..b51d83ced 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt @@ -73,8 +73,8 @@ import org.jacodb.panda.staticvm.cfg.PandaPhiInst as StaticPandaPhiInst private val logger = mu.KotlinLogging.logger {} // TODO: replace with CommonMethod, with CommonInst -context(Traits) class ForwardTaintFlowFunctions( + val traits: Traits, private val graph: ApplicationGraph, val getConfigForMethod: ForwardTaintFlowFunctions.(Method) -> List? = { method -> taintConfigurationFeature?.let { feature -> @@ -114,10 +114,11 @@ class ForwardTaintFlowFunctions( val config = getConfigForMethod(method) if (config != null) { val conditionEvaluator = BasicConditionEvaluator( - EntryPointPositionToValueResolver(method) + traits, + EntryPointPositionToValueResolver(traits, method) ) val actionEvaluator = TaintActionEvaluator( - EntryPointPositionToAccessPathResolver(method) + EntryPointPositionToAccessPathResolver(traits, method) ) // Handle EntryPointSource config items: @@ -140,8 +141,8 @@ class ForwardTaintFlowFunctions( from: CommonExpr, to: CommonValue, ): Collection { - val toPath = to.toPath() - val fromPath = from.toPathOrNull() + val toPath = with(traits) { to.toPath() } + val fromPath = with(traits) { from.toPathOrNull() } if (fromPath != null) { // Adhoc taint array: @@ -271,8 +272,8 @@ class ForwardTaintFlowFunctions( from: CommonValue, to: CommonValue, ): Collection = buildSet { - val fromPath = from.toPath() - val toPath = to.toPath() + val fromPath = with(traits) { from.toPath() } + val toPath = with(traits) { to.toPath() } val tail = (fact.variable - fromPath) ?: return@buildSet val newPath = toPath + tail @@ -314,10 +315,10 @@ class ForwardTaintFlowFunctions( callStatement: Statement, returnSite: Statement, // FIXME: unused? ) = FlowFunction { fact -> - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") - val callee = callExpr.callee + val callee = with(traits) { callExpr.callee } // FIXME: handle taint pass-through on invokedynamic-based String concatenation: if (fact is Tainted @@ -326,10 +327,10 @@ class ForwardTaintFlowFunctions( && callStatement is JcAssignInst ) { for (arg in callExpr.args) { - if (arg.toPath() == fact.variable) { + if (with(traits) { arg.toPath() } == fact.variable) { return@FlowFunction setOf( fact, - fact.copy(variable = callStatement.lhv.toPath()) + fact.copy(variable = with(traits) { callStatement.lhv.toPath() }) ) } } @@ -344,10 +345,11 @@ class ForwardTaintFlowFunctions( if (config != null) { val conditionEvaluator = BasicConditionEvaluator( - CallPositionToValueResolver(callStatement) + traits, + CallPositionToValueResolver(traits, callStatement) ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(callStatement) + CallPositionToAccessPathResolver(traits, callStatement) ) // Handle MethodSource config items: @@ -370,10 +372,12 @@ class ForwardTaintFlowFunctions( if (config != null) { val facts = mutableSetOf() val conditionEvaluator = FactAwareConditionEvaluator( - fact, CallPositionToValueResolver(callStatement) + traits, + CallPositionToValueResolver(traits, callStatement), + fact, ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(callStatement) + CallPositionToAccessPathResolver(traits, callStatement) ) var defaultBehavior = true @@ -424,7 +428,7 @@ class ForwardTaintFlowFunctions( } // FIXME: adhoc for constructors: - if (callee.isConstructor) { + if (with(traits) { callee.isConstructor }) { return@FlowFunction listOf(fact) } @@ -445,14 +449,14 @@ class ForwardTaintFlowFunctions( for (actual in callExpr.args) { // Possibly tainted actual parameter: - if (fact.variable.startsWith(actual.toPathOrNull())) { + if (fact.variable.startsWith(with(traits) { actual.toPathOrNull() })) { return@FlowFunction emptyList() // Will be handled by summary edge } } if (callExpr is CommonInstanceCallExpr) { // Possibly tainted instance: - if (fact.variable.startsWith(callExpr.instance.toPathOrNull())) { + if (fact.variable.startsWith(with(traits) { callExpr.instance.toPathOrNull() })) { return@FlowFunction emptyList() // Will be handled by summary edge } } @@ -461,7 +465,7 @@ class ForwardTaintFlowFunctions( if (callStatement is CommonAssignInst) { // Possibly tainted lhv: - if (fact.variable.startsWith(callStatement.lhv.toPathOrNull())) { + if (fact.variable.startsWith(with(traits) { callStatement.lhv.toPathOrNull() })) { return@FlowFunction emptyList() // Overridden by rhv } } @@ -481,13 +485,13 @@ class ForwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") buildSet { // Transmit facts on arguments (from 'actual' to 'formal'): val actualParams = callExpr.args - val formalParams = getArgumentsOf(callee) + val formalParams = traits.getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll(transmitTaintArgumentActualToFormal(fact, from = actual, to = formal)) } @@ -498,7 +502,7 @@ class ForwardTaintFlowFunctions( transmitTaintInstanceToThis( fact = fact, from = callExpr.instance, - to = callee.thisInstance + to = with(traits) { callee.thisInstance } ) ) } @@ -520,7 +524,7 @@ class ForwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") val callee = graph.methodOf(exitStatement) @@ -528,7 +532,7 @@ class ForwardTaintFlowFunctions( // Transmit facts on arguments (from 'formal' back to 'actual'), if they are passed by-ref: if (fact.variable.isOnHeap) { val actualParams = callExpr.args - val formalParams = getArgumentsOf(callee) + val formalParams = traits.getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentFormalToActual( @@ -545,7 +549,7 @@ class ForwardTaintFlowFunctions( addAll( transmitTaintThisToInstance( fact = fact, - from = callee.thisInstance, + from = with(traits) { callee.thisInstance }, to = callExpr.instance ) ) @@ -567,8 +571,8 @@ class ForwardTaintFlowFunctions( } } -context(Traits) class BackwardTaintFlowFunctions( + val traits: Traits, private val graph: ApplicationGraph, ) : FlowFunctions where Method : CommonMethod, @@ -585,8 +589,8 @@ class BackwardTaintFlowFunctions( from: CommonValue, to: CommonExpr, ): Collection { - val fromPath = from.toPath() - val toPath = to.toPathOrNull() + val fromPath = with(traits) { from.toPath() } + val toPath = with(traits) { to.toPathOrNull() } if (toPath != null) { val tail = fact.variable - fromPath @@ -636,8 +640,8 @@ class BackwardTaintFlowFunctions( from: CommonValue, to: CommonValue, ): Collection = buildSet { - val fromPath = from.toPath() - val toPath = to.toPath() + val fromPath = with(traits) { from.toPath() } + val toPath = with(traits) { to.toPath() } val tail = (fact.variable - fromPath) ?: return@buildSet val newPath = toPath + tail @@ -686,9 +690,9 @@ class BackwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") - val callee = callExpr.callee + val callee = with(traits) { callExpr.callee } if (callee in graph.callees(callStatement)) { @@ -698,14 +702,14 @@ class BackwardTaintFlowFunctions( for (actual in callExpr.args) { // Possibly tainted actual parameter: - if (fact.variable.startsWith(actual.toPathOrNull())) { + if (fact.variable.startsWith(with(traits) { actual.toPathOrNull() })) { return@FlowFunction emptyList() // Will be handled by summary edge } } if (callExpr is CommonInstanceCallExpr) { // Possibly tainted instance: - if (fact.variable.startsWith(callExpr.instance.toPathOrNull())) { + if (fact.variable.startsWith(with(traits) { callExpr.instance.toPathOrNull() })) { return@FlowFunction emptyList() // Will be handled by summary edge } } @@ -714,7 +718,7 @@ class BackwardTaintFlowFunctions( if (callStatement is CommonAssignInst) { // Possibly tainted rhv: - if (fact.variable.startsWith(callStatement.rhv.toPathOrNull())) { + if (fact.variable.startsWith(with(traits) { callStatement.rhv.toPathOrNull() })) { return@FlowFunction emptyList() // Overridden by lhv } } @@ -734,13 +738,13 @@ class BackwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") buildSet { // Transmit facts on arguments (from 'actual' to 'formal'): val actualParams = callExpr.args - val formalParams = getArgumentsOf(callee) + val formalParams = traits.getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll(transmitTaintArgumentActualToFormal(fact, from = actual, to = formal)) } @@ -751,7 +755,7 @@ class BackwardTaintFlowFunctions( transmitTaintInstanceToThis( fact = fact, from = callExpr.instance, - to = callee.thisInstance + to = with(traits) { callee.thisInstance } ) ) } @@ -787,7 +791,7 @@ class BackwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") val callee = graph.methodOf(exitStatement) @@ -795,7 +799,7 @@ class BackwardTaintFlowFunctions( // Transmit facts on arguments (from 'formal' back to 'actual'), if they are passed by-ref: if (fact.variable.isOnHeap) { val actualParams = callExpr.args - val formalParams = getArgumentsOf(callee) + val formalParams = traits.getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentFormalToActual( @@ -812,7 +816,7 @@ class BackwardTaintFlowFunctions( addAll( transmitTaintThisToInstance( fact = fact, - from = callee.thisInstance, + from = with(traits) { callee.thisInstance }, to = callExpr.instance ) ) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt index 641092696..38baaa650 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt @@ -51,13 +51,12 @@ import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds import kotlin.time.DurationUnit -import kotlin.time.ExperimentalTime import kotlin.time.TimeSource private val logger = mu.KotlinLogging.logger {} -context(Traits) open class TaintManager( + val traits: Traits, protected val graph: ApplicationGraph, protected val unitResolver: UnitResolver, private val useBidiRunner: Boolean = false, @@ -91,7 +90,7 @@ open class TaintManager( unitResolver = unitResolver, unit = unit, { manager -> - val analyzer = TaintAnalyzer(graph, getConfigForMethod) + val analyzer = TaintAnalyzer(traits, graph, getConfigForMethod) UniRunner( manager = manager, graph = graph, @@ -102,7 +101,7 @@ open class TaintManager( ) }, { manager -> - val analyzer = BackwardTaintAnalyzer(graph) + val analyzer = BackwardTaintAnalyzer(traits, graph) UniRunner( manager = manager, graph = graph.reversed, @@ -114,7 +113,7 @@ open class TaintManager( } ) } else { - val analyzer = TaintAnalyzer(graph, getConfigForMethod) + val analyzer = TaintAnalyzer(traits, graph, getConfigForMethod) UniRunner( manager = this@TaintManager, graph = graph, @@ -151,7 +150,6 @@ open class TaintManager( } @JvmName("analyze") // needed for Java interop because of inline class (Duration) - @OptIn(ExperimentalTime::class) fun analyze( startMethods: List, timeout: Duration = 3600.seconds, diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt index 148e04b9c..0fdab9d21 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt @@ -24,15 +24,15 @@ import org.jacodb.api.common.CommonMethod import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.api.common.cfg.CommonInst -context(Traits) class UnusedVariableAnalyzer( + val traits: Traits, private val graph: ApplicationGraph, ) : Analyzer, Method, Statement> where Method : CommonMethod, Statement : CommonInst { override val flowFunctions: UnusedVariableFlowFunctions by lazy { - UnusedVariableFlowFunctions(graph) + UnusedVariableFlowFunctions(traits, graph) } private fun isExitPoint(statement: Statement): Boolean { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt index 2f3d9aa08..0397a598d 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt @@ -21,15 +21,14 @@ import org.jacodb.analysis.ifds.FlowFunctions import org.jacodb.analysis.ifds.isOnHeap import org.jacodb.analysis.util.Traits import org.jacodb.api.common.CommonMethod -import org.jacodb.api.common.CommonProject import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.api.common.cfg.CommonAssignInst import org.jacodb.api.common.cfg.CommonInst import org.jacodb.api.jvm.cfg.JcSpecialCallExpr import org.jacodb.api.jvm.cfg.JcStaticCallExpr -context(Traits) class UnusedVariableFlowFunctions( + val traits: Traits, private val graph: ApplicationGraph, ) : FlowFunctions where Method : CommonMethod, @@ -50,7 +49,7 @@ class UnusedVariableFlowFunctions( } if (fact == UnusedVariableZeroFact) { - val toPath = current.lhv.toPath() + val toPath = with(traits) { current.lhv.toPath() } if (!toPath.isOnHeap) { return@FlowFunction setOf(UnusedVariableZeroFact, UnusedVariable(toPath, current)) } else { @@ -59,9 +58,9 @@ class UnusedVariableFlowFunctions( } check(fact is UnusedVariable) - val toPath = current.lhv.toPath() + val toPath = with(traits) { current.lhv.toPath() } val default = if (toPath == fact.variable) emptySet() else setOf(fact) - val fromPath = current.rhv.toPathOrNull() + val fromPath = with(traits) { current.rhv.toPathOrNull() } ?: return@FlowFunction default if (fromPath.isOnHeap || toPath.isOnHeap) { @@ -84,7 +83,7 @@ class UnusedVariableFlowFunctions( callStatement: Statement, calleeStart: Statement, ) = FlowFunction { fact -> - val callExpr = callStatement.getCallExpr() + val callExpr = with(traits) { callStatement.getCallExpr() } ?: error("Call statement should have non-null callExpr") if (fact == UnusedVariableZeroFact) { @@ -95,9 +94,9 @@ class UnusedVariableFlowFunctions( return@FlowFunction buildSet { add(UnusedVariableZeroFact) val callee = graph.methodOf(calleeStart) - val formalParams = getArgumentsOf(callee) + val formalParams = traits.getArgumentsOf(callee) for (formal in formalParams) { - add(UnusedVariable(formal.toPath(), callStatement)) + add(UnusedVariable(with(traits) { formal.toPath() }, callStatement)) } } } diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt index 687465415..1c3369148 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt @@ -54,8 +54,8 @@ import kotlin.time.TimeSource private val logger = mu.KotlinLogging.logger {} -context(Traits) class UnusedVariableManager( + val traits: Traits, private val graph: ApplicationGraph, private val unitResolver: UnitResolver, ) : Manager, Method, Statement> @@ -79,7 +79,7 @@ class UnusedVariableManager( check(unit !in runnerForUnit) { "Runner for $unit already exists" } logger.debug { "Creating a new runner for $unit" } - val analyzer = UnusedVariableAnalyzer(graph) + val analyzer = UnusedVariableAnalyzer(traits, graph) val runner = UniRunner( graph = graph, analyzer = analyzer, @@ -198,7 +198,7 @@ class UnusedVariableManager( if (fact is UnusedVariable) { @Suppress("UNCHECKED_CAST") used.putIfAbsent(fact.initStatement as Statement, false) - if (fact.variable.isUsedAt(inst)) { + if (with(traits) { fact.variable.isUsedAt(inst) }) { used[fact.initStatement] = true } } diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt index b6556ac2b..9f112cacd 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt @@ -58,18 +58,10 @@ import org.jacodb.api.jvm.ext.cfg.callExpr as _callExpr /** * JVM-specific extensions for analysis. - * - * ### Usage: - * ``` - * class MyClass { - * companion object : JcTraits - * } - * ``` */ -interface JcTraits : Traits { - - val cp: JcClasspath - get() = JcTraits.cp +class JcTraits( + val cp: JcClasspath, +) : Traits { override val JcMethod.thisInstance: JcThis get() = _thisInstance @@ -172,11 +164,6 @@ interface JcTraits : Traits { override fun JcInst.getOperands(): List { return operands } - - // Ensure that all methods are default-implemented in the interface itself: - companion object : JcTraits { - override lateinit var cp: JcClasspath - } } val JcMethod.thisInstance: JcThis diff --git a/jacodb-analysis/src/test/java/org/jacodb/analysis/impl/JavaAnalysisApiTest.java b/jacodb-analysis/src/test/java/org/jacodb/analysis/impl/JavaAnalysisApiTest.java index 1bdda4a4b..6617a6cc0 100644 --- a/jacodb-analysis/src/test/java/org/jacodb/analysis/impl/JavaAnalysisApiTest.java +++ b/jacodb-analysis/src/test/java/org/jacodb/analysis/impl/JavaAnalysisApiTest.java @@ -63,7 +63,8 @@ public void testJavaAnalysisApi() throws ExecutionException, InterruptedExceptio .newApplicationGraphForAnalysisAsync(classpath, null) .get(); UnitResolver unitResolver = UnitResolverKt.getMethodUnitResolver(); - TaintManager manager = new TaintManager(JcTraits.Companion, applicationGraph, unitResolver, false, null); + JcTraits traits = new JcTraits(classpath); + TaintManager manager = new TaintManager<>(traits, applicationGraph, unitResolver, false, null); manager.analyze(methodsToAnalyze, toDuration(30, DurationUnit.SECONDS)); } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/BaseAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/BaseAnalysisTest.kt index 3be16d2e9..62ae0fd1c 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/BaseAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/BaseAnalysisTest.kt @@ -42,7 +42,7 @@ private val logger = mu.KotlinLogging.logger {} abstract class BaseAnalysisTest : BaseTest() { - companion object : WithGlobalDB(UnknownClasses), JcTraits { + companion object : WithGlobalDB(UnknownClasses) { fun getJulietClasses( cweNum: Int, @@ -98,8 +98,10 @@ abstract class BaseAnalysisTest : BaseTest() { } else { super.cp } - }.also { - JcTraits.cp = it + } + + val traits: JcTraits by lazy { + JcTraits(cp) } protected val graph: JcApplicationGraph by lazy { diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt index f31050cf5..edb7733f4 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt @@ -63,9 +63,8 @@ import kotlin.test.assertTrue class ConditionEvaluatorTest { - companion object : JcTraits - private val cp = mockk() + private val traits = JcTraits(cp) private val intType: JcPrimitiveType = PredefinedPrimitive(cp, PredefinedPrimitives.Int) private val boolType: JcPrimitiveType = PredefinedPrimitive(cp, PredefinedPrimitives.Boolean) @@ -94,7 +93,7 @@ class ConditionEvaluatorTest { else -> null }.toMaybe() } - private val evaluator: ConditionVisitor = BasicConditionEvaluator(positionResolver) + private val evaluator: ConditionVisitor = BasicConditionEvaluator(traits, positionResolver) @Test fun `True is true`() { @@ -328,8 +327,8 @@ class ConditionEvaluatorTest { @Test fun `FactAwareConditionEvaluator supports ContainsMark`() { - val fact = Tainted(intValue.toPath(), TaintMark("FOO")) - val factAwareEvaluator = FactAwareConditionEvaluator(fact, positionResolver) + val fact = Tainted(with(traits) { intValue.toPath() }, TaintMark("FOO")) + val factAwareEvaluator = FactAwareConditionEvaluator(traits, positionResolver, fact) assertTrue(factAwareEvaluator.visit(ContainsMark(intArg, TaintMark("FOO")))) assertFalse(factAwareEvaluator.visit(ContainsMark(intArg, TaintMark("BAR")))) assertFalse(factAwareEvaluator.visit(ContainsMark(stringArg, TaintMark("FOO")))) diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt index cbb3cb0a0..ea1e697a3 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt @@ -199,7 +199,7 @@ abstract class IfdsNpeTest : BaseAnalysisTest() { private fun findSinks(method: JcMethod): List> { val unitResolver = SingletonUnitResolver - val manager = NpeManager(graph, unitResolver) + val manager = NpeManager(traits, graph, unitResolver) return manager.analyze(listOf(method), timeout = 30.seconds) } @@ -246,4 +246,4 @@ class IfdsNpeSqlTest : IfdsNpeTest() { class IfdsNpeRAMTest : IfdsNpeTest() { companion object : WithRAMDB(Usages, InMemoryHierarchy) -} \ No newline at end of file +} diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt index 7128b4ba8..4c7433a00 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt @@ -62,7 +62,7 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { val method = cp.findClass().declaredMethods.single { it.name == methodName } val methods = listOf(method) val unitResolver = SingletonUnitResolver - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) val sinks = manager.analyze(methods, timeout = 30.seconds) assertTrue(sinks.isNotEmpty()) val sink = sinks.first() @@ -76,7 +76,7 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { fun `test on Juliet's CWE 89`(className: String) { testSingleJulietClass(className) { method -> val unitResolver = SingletonUnitResolver - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) manager.analyze(listOf(method), timeout = 30.seconds) } } @@ -86,7 +86,7 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { val className = "juliet.testcases.CWE89_SQL_Injection.s01.CWE89_SQL_Injection__connect_tcp_execute_01" testSingleJulietClass(className) { method -> val unitResolver = SingletonUnitResolver - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) manager.analyze(listOf(method), timeout = 30.seconds) } } @@ -97,7 +97,7 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { val clazz = cp.findClass(className) val badMethod = clazz.methods.single { it.name == "bad" } val unitResolver = ClassUnitResolver(true) - val manager = TaintManager(graph, unitResolver, useBidiRunner = true) + val manager = TaintManager(traits, graph, unitResolver, useBidiRunner = true) val sinks = manager.analyze(listOf(badMethod), timeout = 30.seconds) assertTrue(sinks.isNotEmpty()) val sink = sinks.first() diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt index 4531d690a..46a7c8880 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt @@ -38,7 +38,7 @@ class IfdsTaintTest : BaseAnalysisTest() { private fun findSinks(method: JcMethod): List> { val unitResolver = SingletonUnitResolver - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) return manager.analyze(listOf(method), timeout = 3000.seconds) } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt index 2c23c22e4..93aeea395 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt @@ -21,7 +21,6 @@ import mu.KotlinLogging import org.jacodb.analysis.ifds.SingletonUnitResolver import org.jacodb.analysis.taint.TaintAnalysisOptions import org.jacodb.analysis.taint.TaintManager -import org.jacodb.analysis.util.JcTraits import org.jacodb.api.jvm.JcClasspath import org.jacodb.api.jvm.ext.findClass import org.jacodb.impl.features.InMemoryHierarchy @@ -49,8 +48,6 @@ class Ifds2UpperBoundTest : BaseAnalysisTest() { } else { super.cp } - }.also { - JcTraits.cp = it } @Test @@ -62,7 +59,7 @@ class Ifds2UpperBoundTest : BaseAnalysisTest() { private inline fun testOneMethod(methodName: String) { val method = cp.findClass().declaredMethods.single { it.name == methodName } val unitResolver = SingletonUnitResolver - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) val sinks = manager.analyze(listOf(method), timeout = 60.seconds) logger.info { "Sinks: ${sinks.size}" } for (sink in sinks) { diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt index ca17c3a28..c6c9a32c0 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt @@ -63,7 +63,7 @@ abstract class IfdsUnusedTest : BaseAnalysisTest() { fun `test on Juliet's CWE 563`(className: String) { testSingleJulietClass(className) { method -> val unitResolver = SingletonUnitResolver - val manager = UnusedVariableManager(graph, unitResolver) + val manager = UnusedVariableManager(traits, graph, unitResolver) manager.analyze(listOf(method), timeout = 30.seconds) } } @@ -75,7 +75,7 @@ abstract class IfdsUnusedTest : BaseAnalysisTest() { val clazz = cp.findClass(className) val badMethod = clazz.methods.single { it.name == "bad" } val unitResolver = SingletonUnitResolver - val manager = UnusedVariableManager(graph, unitResolver) + val manager = UnusedVariableManager(traits, graph, unitResolver) val sinks = manager.analyze(listOf(badMethod), timeout = 30.seconds) Assertions.assertTrue(sinks.isNotEmpty()) } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt index a5a1f50af..9f57e8cdc 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt @@ -21,7 +21,6 @@ import org.jacodb.analysis.ifds.SingletonUnitResolver import org.jacodb.analysis.npe.NpeManager import org.jacodb.analysis.taint.TaintManager import org.jacodb.analysis.unused.UnusedVariableManager -import org.jacodb.analysis.util.JcTraits import org.jacodb.api.jvm.JcClasspath import org.jacodb.api.jvm.ext.findClass import org.jacodb.taint.configuration.TaintConfigurationFeature @@ -47,8 +46,6 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { } else { super.cp } - }.also { - JcTraits.cp = it } @Test @@ -56,7 +53,7 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { val clazz = cp.findClass() val methods = clazz.declaredMethods val unitResolver = SingletonUnitResolver - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) val sinks = manager.analyze(methods, timeout = 60.seconds) logger.info { "Vulnerabilities found: ${sinks.size}" } } @@ -66,7 +63,7 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { val clazz = cp.findClass() val methods = clazz.declaredMethods val unitResolver = SingletonUnitResolver - val manager = NpeManager(graph, unitResolver) + val manager = NpeManager(traits, graph, unitResolver) val sinks = manager.analyze(methods, timeout = 60.seconds) logger.info { "Vulnerabilities found: ${sinks.size}" } } @@ -76,7 +73,7 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { val clazz = cp.findClass() val methods = clazz.declaredMethods val unitResolver = SingletonUnitResolver - val manager = UnusedVariableManager(graph, unitResolver) + val manager = UnusedVariableManager(traits, graph, unitResolver) val sinks = manager.analyze(methods, timeout = 60.seconds) logger.info { "Unused variables found: ${sinks.size}" } } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt index f890d8e9f..6058b5cd9 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt @@ -51,9 +51,9 @@ import org.junit.jupiter.api.Test open class TaintFlowFunctionsTest : BaseTest() { - companion object : WithDB(Usages, InMemoryHierarchy), JcTraits + companion object : WithDB(Usages, InMemoryHierarchy) - override val cp: JcClasspath = runBlocking { + final override val cp: JcClasspath = runBlocking { val configFileName = "config_test.json" val configResource = this.javaClass.getResourceAsStream("/$configFileName") if (configResource != null) { @@ -63,14 +63,14 @@ open class TaintFlowFunctionsTest : BaseTest() { } else { super.cp } - }.also { - JcTraits.cp = it } + private val traits = JcTraits(cp) + private val graph: JcApplicationGraph = mockk { every { cp } returns this@TaintFlowFunctionsTest.cp every { callees(any()) } answers { - sequenceOf(arg(0).callExpr!!.callee) + sequenceOf(with(traits) { arg(0).callExpr!!.callee }) } every { methodOf(any()) } answers { arg(0).location.method @@ -102,10 +102,10 @@ open class TaintFlowFunctionsTest : BaseTest() { @Test fun `test obtain start facts`() { - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val facts = flowSpace.obtainPossibleStartFacts(testMethod).toList() - val arg0 = getArgument(testMethod.parameters[0])!! - val arg0Taint = Tainted(arg0.toPath(), TaintMark("EXAMPLE")) + val arg0 = traits.getArgument(testMethod.parameters[0])!! + val arg0Taint = Tainted(with(traits) { arg0.toPath() }, TaintMark("EXAMPLE")) Assertions.assertEquals(listOf(TaintZeroFact, arg0Taint), facts) } @@ -115,10 +115,10 @@ open class TaintFlowFunctionsTest : BaseTest() { val x: JcLocal = JcLocalVar(1, "x", stringType) val y: JcLocal = JcLocalVar(2, "y", stringType) val inst = JcAssignInst(location = mockk(), lhv = x, rhv = y) - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val f = flowSpace.obtainSequentFlowFunction(inst, next = mockk()) - val yTaint = Tainted(y.toPath(), TaintMark("TAINT")) - val xTaint = Tainted(x.toPath(), TaintMark("TAINT")) + val yTaint = Tainted(with(traits) { y.toPath() }, TaintMark("TAINT")) + val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("TAINT")) val facts = f.compute(yTaint).toList() Assertions.assertEquals(listOf(yTaint, xTaint), facts) } @@ -130,9 +130,9 @@ open class TaintFlowFunctionsTest : BaseTest() { val callStatement = JcAssignInst(location = mockk(), lhv = x, rhv = mockk { every { method.method } returns testMethod }) - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val f = flowSpace.obtainCallToReturnSiteFlowFunction(callStatement, returnSite = mockk()) - val xTaint = Tainted(x.toPath(), TaintMark("EXAMPLE")) + val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("EXAMPLE")) val facts = f.compute(TaintZeroFact).toList() Assertions.assertEquals(listOf(TaintZeroFact, xTaint), facts) } @@ -145,9 +145,9 @@ open class TaintFlowFunctionsTest : BaseTest() { every { method.method } returns testMethod every { args } returns listOf(x) }) - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val f = flowSpace.obtainCallToReturnSiteFlowFunction(callStatement, returnSite = mockk()) - val xTaint = Tainted(x.toPath(), TaintMark("REMOVE")) + val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("REMOVE")) val facts = f.compute(xTaint).toList() Assertions.assertTrue(facts.isEmpty()) } @@ -161,14 +161,14 @@ open class TaintFlowFunctionsTest : BaseTest() { every { method.method } returns testMethod every { args } returns listOf(x) }) - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val f = flowSpace.obtainCallToReturnSiteFlowFunction(callStatement, returnSite = mockk()) - val xTaint = Tainted(x.toPath(), TaintMark("COPY")) - val yTaint = Tainted(y.toPath(), TaintMark("COPY")) + val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("COPY")) + val yTaint = Tainted(with(traits) { y.toPath() }, TaintMark("COPY")) val facts = f.compute(xTaint).toList() Assertions.assertEquals(listOf(xTaint, yTaint), facts) // copy from x to y val other: JcLocal = JcLocalVar(10, "other", stringType) - val otherTaint = Tainted(other.toPath(), TaintMark("OTHER")) + val otherTaint = Tainted(with(traits) { other.toPath() }, TaintMark("OTHER")) val facts2 = f.compute(otherTaint).toList() Assertions.assertEquals(listOf(otherTaint), facts2) // pass-through } @@ -181,19 +181,19 @@ open class TaintFlowFunctionsTest : BaseTest() { every { method.method } returns testMethod every { args } returns listOf(x) }) - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val f = flowSpace.obtainCallToStartFlowFunction(callStatement, calleeStart = mockk { every { location } returns mockk { every { method } returns testMethod } }) - val xTaint = Tainted(x.toPath(), TaintMark("TAINT")) - val arg0: JcArgument = getArgument(testMethod.parameters[0])!! - val arg0Taint = Tainted(arg0.toPath(), TaintMark("TAINT")) + val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("TAINT")) + val arg0: JcArgument = traits.getArgument(testMethod.parameters[0])!! + val arg0Taint = Tainted(with(traits) { arg0.toPath() }, TaintMark("TAINT")) val facts = f.compute(xTaint).toList() Assertions.assertEquals(listOf(arg0Taint), facts) val other: JcLocal = JcLocalVar(10, "other", stringType) - val otherTaint = Tainted(other.toPath(), TaintMark("TAINT")) + val otherTaint = Tainted(with(traits) { other.toPath() }, TaintMark("TAINT")) val facts2 = f.compute(otherTaint).toList() Assertions.assertTrue(facts2.isEmpty()) } @@ -209,10 +209,10 @@ open class TaintFlowFunctionsTest : BaseTest() { val exitStatement = JcReturnInst(location = mockk { every { method } returns testMethod }, returnValue = y) - val flowSpace = ForwardTaintFlowFunctions(graph) + val flowSpace = ForwardTaintFlowFunctions(traits, graph) val f = flowSpace.obtainExitToReturnSiteFlowFunction(callStatement, returnSite = mockk(), exitStatement) - val yTaint = Tainted(y.toPath(), TaintMark("TAINT")) - val xTaint = Tainted(x.toPath(), TaintMark("TAINT")) + val yTaint = Tainted(with(traits) { y.toPath() }, TaintMark("TAINT")) + val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("TAINT")) val facts = f.compute(yTaint).toList() Assertions.assertEquals(listOf(xTaint), facts) } diff --git a/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt b/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt index 7591cbc26..55f5fbe0e 100644 --- a/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt +++ b/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt @@ -62,9 +62,10 @@ data class AnalysisConfig(val analyses: Map) fun launchAnalysesByConfig( config: AnalysisConfig, + traits: JcTraits, graph: JcApplicationGraph, methods: List, -): List>> = with(JcTraits) { +): List>> { return config.analyses.mapNotNull { (analysis, options) -> val unitResolver = options["UnitResolver"]?.let { UnitResolver.getByName(it) @@ -72,17 +73,17 @@ fun launchAnalysesByConfig( when (analysis) { "NPE" -> { - val manager = NpeManager(graph, unitResolver) + val manager = NpeManager(traits, graph, unitResolver) manager.analyze(methods, timeout = 60.seconds).map { it.toSarif(manager.vulnerabilityTraceGraph(it)) } } "Unused" -> { - val manager = UnusedVariableManager(graph, unitResolver) + val manager = UnusedVariableManager(traits, graph, unitResolver) manager.analyze(methods, timeout = 60.seconds).map { it.toSarif() } } "SQL" -> { - val manager = TaintManager(graph, unitResolver) + val manager = TaintManager(traits, graph, unitResolver) manager.analyze(methods, timeout = 60.seconds).map { it.toSarif(manager.vulnerabilityTraceGraph(it)) } } @@ -156,7 +157,6 @@ fun main(args: Array) { } jacodb.classpath(classpathAsFiles) } - JcTraits.cp = cp val startClassesAsList = startClasses.split(";") val startJcClasses = ConcurrentHashMap.newKeySet() @@ -169,11 +169,12 @@ fun main(args: Array) { }).get() val startJcMethods = startJcClasses.flatMap { it.declaredMethods }.filter { !it.isPrivate } + val traits = JcTraits(cp) val graph = runBlocking { cp.newApplicationGraphForAnalysis() } - val vulnerabilities = launchAnalysesByConfig(config, graph, startJcMethods).flatten() + val vulnerabilities = launchAnalysesByConfig(config, traits, graph, startJcMethods).flatten() val report = sarifReportFromVulnerabilities(vulnerabilities) val prettyJson = Json { prettyPrint = true diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt index 158659ea0..e7ad90720 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt @@ -53,7 +53,9 @@ private val logger = mu.KotlinLogging.logger {} @Disabled("Have several issues with EtsIR") class EtsIfds { - companion object : EtsTraits { + companion object { + private val traits = EtsTraits + private const val BASE_PATH = "/etsir/samples" private fun loadSample(programName: String): EtsFile { @@ -96,6 +98,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -141,6 +144,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -205,6 +209,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -245,6 +250,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -283,6 +289,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -329,6 +336,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -408,6 +416,7 @@ class EtsIfds { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt index 633534a09..2159a0875 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt @@ -16,9 +16,6 @@ package org.jacodb.ets.test -import org.jacodb.ets.test.utils.getConfigForMethod -import org.jacodb.ets.test.utils.loadEtsFileFromResource -import org.jacodb.ets.test.utils.loadRules import org.jacodb.analysis.ifds.SingletonUnit import org.jacodb.analysis.ifds.UnitResolver import org.jacodb.analysis.taint.TaintManager @@ -30,8 +27,11 @@ import org.jacodb.ets.graph.EtsApplicationGraphImpl import org.jacodb.ets.model.EtsFile import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsScene +import org.jacodb.ets.test.utils.getConfigForMethod import org.jacodb.ets.test.utils.getResourcePath import org.jacodb.ets.test.utils.getResourceStream +import org.jacodb.ets.test.utils.loadEtsFileFromResource +import org.jacodb.ets.test.utils.loadRules import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIf import java.nio.file.Files @@ -50,7 +50,9 @@ class EtsProjectAnalysis { private var totalPathEdges = 0 private var totalSinks: MutableList> = mutableListOf() - companion object : EtsTraits { + companion object { + private val traits = EtsTraits + private const val SOURCE_PROJECT_PATH = "/projects/applications_app_samples/source/applications_app_samples/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc" private const val PROJECT_PATH = "/projects/applications_app_samples/etsir/ast/ArkTSDistributedCalc" private const val START_PATH = "/entry/src/main/ets" @@ -140,6 +142,7 @@ class EtsProjectAnalysis { val graph = EtsApplicationGraphImpl(project) val unitResolver = UnitResolver { SingletonUnit } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = { method -> getConfigForMethod(method, rules) }, diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt index 2826c4207..556553958 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt @@ -48,7 +48,9 @@ private val logger = mu.KotlinLogging.logger {} class EtsTaintAnalysisTest { - companion object : EtsTraits { + companion object { + private val traits = EtsTraits + private const val BASE_PATH = "/samples/etsir/ast" private const val DECOMPILED_PATH = "/decompiled" @@ -109,6 +111,7 @@ class EtsTaintAnalysisTest { val unitResolver = UnitResolver { SingletonUnit } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, diff --git a/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt b/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt index e4bb096b5..7bb71c3c2 100644 --- a/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt +++ b/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt @@ -44,7 +44,9 @@ private val logger = mu.KotlinLogging.logger {} class PandaDemoCases { - companion object : PandaStaticTraits + companion object { + private val traits = PandaStaticTraits + } private fun loadProject(path: String): PandaProject { val program = loadProgram("/$path") @@ -78,6 +80,7 @@ class PandaDemoCases { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -112,7 +115,7 @@ class PandaDemoCases { ), ) ) - if (method.isConstructor && method.enclosingClass.name == "escompat.ArrayBuffer") add( + if (with(traits) { method.isConstructor } && method.enclosingClass.name == "escompat.ArrayBuffer") add( TaintMethodSink( method = method, ruleNote = "ArrayBuffer constructor", @@ -125,6 +128,7 @@ class PandaDemoCases { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, diff --git a/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt b/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt index fc358d299..f612df1bc 100644 --- a/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt +++ b/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt @@ -46,7 +46,9 @@ private val logger = mu.KotlinLogging.logger {} class PandaIfdsTest { - companion object : PandaStaticTraits + companion object { + private val traits = PandaStaticTraits + } private fun stdlibAvailable() = EtsStdlib.stdlibAvailable() @@ -114,6 +116,7 @@ class PandaIfdsTest { rules.ifEmpty { null } } val manager = TaintManager( + traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, From f0a304fcd93efb404454ea01fea1a7ccde809a2e Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 21 Nov 2024 14:39:49 +0300 Subject: [PATCH 104/120] Update AA version on CI --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e2be50e21..f2d4e5b3b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-11-06" + BRANCH="neo/2024-11-21" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break From a517ef782b75663e50e75335221db8a520fc6302 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 22 Nov 2024 16:08:34 +0300 Subject: [PATCH 105/120] Update model: type parameters, AliasType, AnnotationType --- .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 8 +++-- .../main/kotlin/org/jacodb/ets/dto/Model.kt | 4 +-- .../kotlin/org/jacodb/ets/dto/Signatures.kt | 10 ++++++ .../main/kotlin/org/jacodb/ets/dto/Types.kt | 34 ++++++++++++++++++- .../org/jacodb/ets/utils/EtsFileDtoToDot.kt | 10 +++--- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 5c97bad2d..38fd338fc 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -582,7 +582,7 @@ fun convertToEtsClass(classDto: ClassDto): EtsClass { val methods = methodDtos.map { convertToEtsMethod(it) } val ctor = convertToEtsMethod(ctorDto) - val typeParameters = classDto.typeParameters.map { convertToEtsType(it) } + val typeParameters = classDto.typeParameters?.map { convertToEtsType(it) } ?: emptyList() val modifiers = EtsModifiers(classDto.modifiers) val decorators = classDto.decorators.map { convertToEtsDecorator(it) } @@ -615,6 +615,10 @@ fun convertToEtsType(type: TypeDto): EtsType { } } + is AliasTypeDto -> EtsUnknownType // TODO: EtsAliasType + is AnnotationNamespaceTypeDto -> EtsUnknownType // TODO: EtsAnnotationNamespaceType + is AnnotationTypeQueryTypeDto -> EtsUnknownType // TODO: EtsAnnotationTypeQueryType + AnyTypeDto -> EtsAnyType is ArrayTypeDto -> EtsArrayType( @@ -762,7 +766,7 @@ fun convertToEtsMethodSignature(method: MethodSignatureDto): EtsMethodSignature fun convertToEtsMethod(method: MethodDto): EtsMethod { val signature = convertToEtsMethodSignature(method.signature) - val typeParameters = method.typeParameters.map { convertToEtsType(it) } + val typeParameters = method.typeParameters?.map { convertToEtsType(it) } ?: emptyList() val modifiers = EtsModifiers(method.modifiers) val decorators = method.decorators.map { convertToEtsDecorator(it) } if (method.body != null) { diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt index c6978c0a9..3cb36992a 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Model.kt @@ -56,7 +56,7 @@ data class ClassDto( val signature: ClassSignatureDto, val modifiers: Int, val decorators: List, - val typeParameters: List, + val typeParameters: List? = null, val superClassName: String?, val implementedInterfaceNames: List, val fields: List, @@ -77,7 +77,7 @@ data class MethodDto( val signature: MethodSignatureDto, val modifiers: Int, val decorators: List, - val typeParameters: List, + val typeParameters: List? = null, val body: BodyDto? = null, ) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt index b4a3879f1..beb1b5375 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Signatures.kt @@ -92,3 +92,13 @@ data class MethodParameterDto( return "$name: $type" } } + +@Serializable +data class LocalSignatureDto( + val name: String, + val method: MethodSignatureDto, +) { + override fun toString(): String { + return "${method}#${name}" + } +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt index df941b905..ed395210b 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Types.kt @@ -20,7 +20,6 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonClassDiscriminator -import kotlin.math.sign @Serializable @OptIn(ExperimentalSerializationApi::class) @@ -239,6 +238,39 @@ data class GenericTypeDto( } } +@Serializable +@SerialName("AliasType") +data class AliasTypeDto( + val name: String, + val originalType: TypeDto, + val signature: LocalSignatureDto, +) : TypeDto { + override fun toString(): String { + return "$name = $originalType" + } +} + +@Serializable +@SerialName("AnnotationNamespaceType") +data class AnnotationNamespaceTypeDto( + val originType: String, + val namespaceSignature: NamespaceSignatureDto, +) : TypeDto { + override fun toString(): String { + return originType + } +} + +@Serializable +@SerialName("AnnotationTypeQueryType") +data class AnnotationTypeQueryTypeDto( + val originType: String, +) : TypeDto { + override fun toString(): String { + return originType + } +} + @Serializable @SerialName("UNKNOWN_TYPE") data class AbsolutelyUnknownTypeDto( diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt index 1587056f7..8c6253132 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/EtsFileDtoToDot.kt @@ -48,8 +48,9 @@ fun EtsFileDto.toDot(useLR: Boolean = false): String { val labelLines: MutableList = mutableListOf() run { val name = clazz.signature.name - val generics = if (clazz.typeParameters.isNotEmpty()) { - "<${clazz.typeParameters.joinToString()}>" + val typeParameters = clazz.typeParameters.orEmpty() + val generics = if (typeParameters.isNotEmpty()) { + "<${typeParameters.joinToString()}>" } else { "" } @@ -65,8 +66,9 @@ fun EtsFileDto.toDot(useLR: Boolean = false): String { clazz.methods.forEach { method -> val name = method.signature.name val params = method.signature.parameters.joinToString() - val generics = if (method.typeParameters.isNotEmpty()) { - "<${method.typeParameters.joinToString()}>" + val typeParameters = method.typeParameters.orEmpty() + val generics = if (typeParameters.isNotEmpty()) { + "<${typeParameters.joinToString()}>" } else { "" } From 78ac19fea3d70840a23896eb0ffa320a79566b80 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 22 Nov 2024 16:10:26 +0300 Subject: [PATCH 106/120] Use `-t 1` in prepare_projects script --- jacodb-ets/src/test/resources/prepare_projects.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/test/resources/prepare_projects.sh b/jacodb-ets/src/test/resources/prepare_projects.sh index 6e3dad460..57c52ef82 100644 --- a/jacodb-ets/src/test/resources/prepare_projects.sh +++ b/jacodb-ets/src/test/resources/prepare_projects.sh @@ -87,8 +87,8 @@ function prepare_module() { ln -srfT "$ROOT/src/main/ets" $SRC echo "Serializing..." # TODO: add switch for using npx/node - # npx ts-node --files --transpileOnly $SCRIPT_TS -p $SRC $ETSIR -v - node $SCRIPT_JS -p $SRC $ETSIR -v -t 2 + # npx ts-node --files --transpileOnly $SCRIPT_TS -p $SRC $ETSIR -v -t 1 + node $SCRIPT_JS -p $SRC $ETSIR -v -t 1 } ( From d65094b2b3c1ac649f858ededc3838e06b04edab Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Thu, 28 Nov 2024 17:46:39 +0300 Subject: [PATCH 107/120] Traverse all namespaces recursively to get ALL classes --- jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsFile.kt | 4 ++++ .../src/main/kotlin/org/jacodb/ets/model/EtsNamespace.kt | 5 ++++- jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsFile.kt index d5290b362..d9c363c2d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsFile.kt @@ -26,6 +26,10 @@ class EtsFile( val projectName: String get() = signature.projectName + val allClasses: List by lazy { + classes + namespaces.flatMap { it.allClasses } + } + override fun toString(): String { return name } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsNamespace.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsNamespace.kt index 5d982f57d..17ab2d0bc 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsNamespace.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsNamespace.kt @@ -20,4 +20,7 @@ class EtsNamespace( val signature: EtsNamespaceSignature, val classes: List, val namespaces: List, -) +) { + val allClasses: List + get() = classes + namespaces.flatMap { it.allClasses } +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt index 22be9003d..179473fc1 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt @@ -22,5 +22,5 @@ class EtsScene( val files: List, ) : CommonProject { val classes: List - get() = files.flatMap { it.classes } + get() = files.flatMap { it.allClasses } } From bb8ed47eaa1fb1a2f6c9f169de080490907525aa Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 15:45:30 +0300 Subject: [PATCH 108/120] Revert context receivers for Traits --- .../org/jacodb/analysis/config/Condition.kt | 20 ++-- .../org/jacodb/analysis/config/Position.kt | 29 +++--- .../org/jacodb/analysis/npe/NpeAnalyzers.kt | 13 ++- .../jacodb/analysis/npe/NpeFlowFunctions.kt | 79 +++++++--------- .../org/jacodb/analysis/npe/NpeManager.kt | 6 +- .../jacodb/analysis/taint/TaintAnalyzers.kt | 23 +++-- .../analysis/taint/TaintFlowFunctions.kt | 86 ++++++++--------- .../org/jacodb/analysis/taint/TaintManager.kt | 10 +- .../analysis/unused/UnusedVariableAnalyzer.kt | 4 +- .../unused/UnusedVariableFlowFunctions.kt | 15 +-- .../analysis/unused/UnusedVariableManager.kt | 6 +- .../org/jacodb/analysis/util/EtsTraits.kt | 9 +- .../org/jacodb/analysis/util/JcTraits.kt | 2 +- .../analysis/impl/ConditionEvaluatorTest.kt | 10 +- .../org/jacodb/analysis/impl/IfdsNpeTest.kt | 4 +- .../org/jacodb/analysis/impl/IfdsSqlTest.kt | 16 +++- .../org/jacodb/analysis/impl/IfdsTaintTest.kt | 4 +- .../impl/IfdsUntrustedLoopBoundTest.kt | 4 +- .../jacodb/analysis/impl/IfdsUnusedTest.kt | 11 ++- .../analysis/impl/JodaDateTimeAnalysisTest.kt | 28 ++---- .../analysis/impl/TaintFlowFunctionsTest.kt | 69 +++++++------- .../src/main/kotlin/org/jacodb/cli/main.kt | 18 ++-- .../kotlin/org/jacodb/ets/test/EtsIfds.kt | 94 ++++++++++--------- .../org/jacodb/ets/test/EtsProjectAnalysis.kt | 18 ++-- .../jacodb/ets/test/EtsTaintAnalysisTest.kt | 16 ++-- .../src/test/kotlin/panda/PandaDemoCases.kt | 8 +- .../src/test/kotlin/panda/PandaIfdsTest.kt | 5 +- 27 files changed, 300 insertions(+), 307 deletions(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt index d9cc58377..376fae3db 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Condition.kt @@ -45,8 +45,8 @@ import org.jacodb.taint.configuration.TypeMatches // TODO: replace 'JcInt' with 'CommonInt', etc +context(Traits) open class BasicConditionEvaluator( - val traits: Traits, internal val positionResolver: PositionResolver>, ) : ConditionVisitor { @@ -78,35 +78,35 @@ open class BasicConditionEvaluator( error("Unexpected condition: $condition") } - override fun visit(condition: IsConstant): Boolean = with(traits) { + override fun visit(condition: IsConstant): Boolean { positionResolver.resolve(condition.position).onSome { return it.isConstant() } return false } - override fun visit(condition: ConstantEq): Boolean = with(traits) { + override fun visit(condition: ConstantEq): Boolean { positionResolver.resolve(condition.position).onSome { value -> return value.eqConstant(condition.value) } return false } - override fun visit(condition: ConstantLt): Boolean = with(traits) { + override fun visit(condition: ConstantLt): Boolean { positionResolver.resolve(condition.position).onSome { value -> return value.ltConstant(condition.value) } return false } - override fun visit(condition: ConstantGt): Boolean = with(traits) { + override fun visit(condition: ConstantGt): Boolean { positionResolver.resolve(condition.position).onSome { value -> return value.gtConstant(condition.value) } return false } - override fun visit(condition: ConstantMatches): Boolean = with(traits){ + override fun visit(condition: ConstantMatches): Boolean { positionResolver.resolve(condition.position).onSome { value -> return value.matches(condition.pattern) } @@ -135,13 +135,13 @@ open class BasicConditionEvaluator( } } +context(Traits) class FactAwareConditionEvaluator( - traits: Traits, - positionResolver: PositionResolver>, private val fact: Tainted, -) : BasicConditionEvaluator(traits, positionResolver) { + positionResolver: PositionResolver>, +) : BasicConditionEvaluator(positionResolver) { - override fun visit(condition: ContainsMark): Boolean = with(traits) { + override fun visit(condition: ContainsMark): Boolean { if (fact.mark != condition.mark) return false positionResolver.resolve(condition.position).onSome { value -> val variable = value.toPath() diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt index 6fc74c68b..347e72497 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/config/Position.kt @@ -35,16 +35,14 @@ import org.jacodb.taint.configuration.Result import org.jacodb.taint.configuration.ResultAnyElement import org.jacodb.taint.configuration.This +context(Traits) class CallPositionToAccessPathResolver( - val traits: Traits, private val callStatement: CommonInst, ) : PositionResolver> { - private val callExpr = with(traits) { - callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") - } + private val callExpr = callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") - override fun resolve(position: Position): Maybe = with(traits) {when (position) { + override fun resolve(position: Position): Maybe = when (position) { AnyArgument -> Maybe.none() is Argument -> callExpr.args[position.index].toPathOrNull().toMaybe() This -> (callExpr as? CommonInstanceCallExpr)?.instance?.toPathOrNull().toMaybe() @@ -52,17 +50,14 @@ class CallPositionToAccessPathResolver( ResultAnyElement -> (callStatement as? CommonAssignInst)?.lhv?.toPathOrNull().toMaybe() .fmap { it + ElementAccessor } } - } } +context(Traits) class CallPositionToValueResolver( - val traits: Traits, private val callStatement: CommonInst, ) : PositionResolver> { - private val callExpr = with(traits) { - callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") - } + private val callExpr = callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") override fun resolve(position: Position): Maybe = when (position) { AnyArgument -> Maybe.none() @@ -73,11 +68,11 @@ class CallPositionToValueResolver( } } +context(Traits) class EntryPointPositionToValueResolver( - val traits: Traits, private val method: CommonMethod, ) : PositionResolver> { - override fun resolve(position: Position): Maybe = with(traits) {when (position) { + override fun resolve(position: Position): Maybe = when (position) { This -> Maybe.some(method.thisInstance) is Argument -> { @@ -87,14 +82,13 @@ class EntryPointPositionToValueResolver( AnyArgument, Result, ResultAnyElement -> error("Unexpected $position") } - } } +context(Traits) class EntryPointPositionToAccessPathResolver( - val traits: Traits, private val method: CommonMethod, ) : PositionResolver> { - override fun resolve(position: Position): Maybe = with(traits){ when (position) { + override fun resolve(position: Position): Maybe = when (position) { This -> method.thisInstance.toPathOrNull().toMaybe() is Argument -> { @@ -104,5 +98,4 @@ class EntryPointPositionToAccessPathResolver( AnyArgument, Result, ResultAnyElement -> error("Unexpected $position") } - } } diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt index 4c87032c3..aef9e63a7 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeAnalyzers.kt @@ -40,15 +40,15 @@ import org.jacodb.taint.configuration.TaintMethodSink private val logger = mu.KotlinLogging.logger {} +context(Traits) class NpeAnalyzer( - private val traits: Traits, private val graph: ApplicationGraph, ) : Analyzer, Method, Statement> where Method : CommonMethod, Statement : CommonInst { override val flowFunctions: ForwardNpeFlowFunctions by lazy { - ForwardNpeFlowFunctions(traits, graph) + ForwardNpeFlowFunctions(graph) } private val taintConfigurationFeature: TaintConfigurationFeature? @@ -66,7 +66,7 @@ class NpeAnalyzer( } if (edge.to.fact is Tainted && edge.to.fact.mark == TaintMark.NULLNESS) { - if (with(traits) { edge.to.fact.variable.isDereferencedAt(edge.to.statement) }) { + if (edge.to.fact.variable.isDereferencedAt(edge.to.statement)) { val message = "NPE" // TODO val vulnerability = TaintVulnerability(message, sink = edge.to) logger.info { @@ -78,8 +78,8 @@ class NpeAnalyzer( } run { - val callExpr = with(traits) { edge.to.statement.getCallExpr() } ?: return@run - val callee = with(traits) { callExpr.callee } + val callExpr = edge.to.statement.getCallExpr() ?: return@run + val callee = callExpr.callee val config = taintConfigurationFeature?.let { feature -> if (callee is JcMethod) { @@ -98,9 +98,8 @@ class NpeAnalyzer( // Determine whether 'edge.to' is a sink via config: val conditionEvaluator = FactAwareConditionEvaluator( - traits, - CallPositionToValueResolver(traits, edge.to.statement), edge.to.fact, + CallPositionToValueResolver(edge.to.statement), ) for (item in config.filterIsInstance()) { if (item.condition.accept(conditionEvaluator)) { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt index 5b9346e93..505a4b489 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeFlowFunctions.kt @@ -74,8 +74,8 @@ import org.jacodb.taint.configuration.TaintPassThrough private val logger = mu.KotlinLogging.logger {} +context(Traits) class ForwardNpeFlowFunctions( - private val traits: Traits, private val graph: ApplicationGraph, ) : FlowFunctions where Method : CommonMethod, @@ -103,7 +103,7 @@ class ForwardNpeFlowFunctions( for (p in method.parameters.filter { it.isNullable != false }) { val t = (graph as JcApplicationGraph).cp.findType(p.type.typeName) val arg = JcArgument.of(p.index, p.name, t) - val path = with(traits) { arg.toPath() } + val path = arg.toPath() add(Tainted(path, TaintMark.NULLNESS)) } } @@ -124,8 +124,8 @@ class ForwardNpeFlowFunctions( } } if (config != null) { - val conditionEvaluator = BasicConditionEvaluator(traits, EntryPointPositionToValueResolver(traits, method)) - val actionEvaluator = TaintActionEvaluator(EntryPointPositionToAccessPathResolver(traits,method)) + val conditionEvaluator = BasicConditionEvaluator(EntryPointPositionToValueResolver(method)) + val actionEvaluator = TaintActionEvaluator(EntryPointPositionToAccessPathResolver(method)) // Handle EntryPointSource config items: for (item in config.filterIsInstance()) { @@ -147,8 +147,8 @@ class ForwardNpeFlowFunctions( from: JcExpr, to: CommonValue, ): Collection { - val toPath = with(traits){ to.toPath() } - val fromPath = with(traits) { from.toPathOrNull() } + val toPath = to.toPath() + val fromPath = from.toPathOrNull() if (fact.mark == TaintMark.NULLNESS) { // TODO: consider @@ -208,7 +208,7 @@ class ForwardNpeFlowFunctions( inst: Statement, ): Collection = buildList { if (inst is CommonAssignInst) { - val toPath = with(traits) { inst.lhv.toPath() } + val toPath = inst.lhv.toPath() val from = inst.rhv if (from is JcNullConstant || (from is JcCallExpr && from.method.method.isNullable == true)) { add(Tainted(toPath, TaintMark.NULLNESS)) @@ -224,9 +224,9 @@ class ForwardNpeFlowFunctions( get() { val expr = condition return if (expr.rhv is JcNullConstant) { - with(traits) { expr.lhv.toPathOrNull() } + expr.lhv.toPathOrNull() } else if (expr.lhv is JcNullConstant) { - with(traits) { expr.rhv.toPathOrNull() } + expr.rhv.toPathOrNull() } else { null } @@ -237,7 +237,7 @@ class ForwardNpeFlowFunctions( next: Statement, ) = FlowFunction { fact -> if (fact is Tainted && fact.mark == TaintMark.NULLNESS) { - if ( with(traits){ fact.variable.isDereferencedAt(current) }) { + if (fact.variable.isDereferencedAt(current)) { return@FlowFunction emptySet() } } @@ -290,13 +290,13 @@ class ForwardNpeFlowFunctions( to: CommonValue, ): Collection = buildSet { if (fact.mark == TaintMark.NULLNESS) { - if ( with(traits){ fact.variable.isDereferencedAt(at) }) { + if (fact.variable.isDereferencedAt(at)) { return@buildSet } } - val fromPath = with(traits){ from.toPath() } - val toPath = with(traits){ to.toPath() } + val fromPath = from.toPath() + val toPath = to.toPath() val tail = (fact.variable - fromPath) ?: return@buildSet val newPath = toPath + tail @@ -344,17 +344,15 @@ class ForwardNpeFlowFunctions( returnSite: Statement, // FIXME: unused? ) = FlowFunction { fact -> if (fact is Tainted && fact.mark == TaintMark.NULLNESS) { - if ( with(traits){ fact.variable.isDereferencedAt(callStatement) }) { + if (fact.variable.isDereferencedAt(callStatement)) { return@FlowFunction emptySet() } } - val callExpr = with(traits) { - callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") - } + val callExpr = callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") - val callee = with(traits){ callExpr.callee } + val callee = callExpr.callee // FIXME: handle taint pass-through on invokedynamic-based String concatenation: if (fact is Tainted @@ -363,10 +361,10 @@ class ForwardNpeFlowFunctions( && callStatement is JcAssignInst ) { for (arg in callExpr.args) { - if ( with(traits) { arg.toPath() } == fact.variable) { + if (arg.toPath() == fact.variable) { return@FlowFunction setOf( fact, - fact.copy(variable = with(traits){callStatement.lhv.toPath()}) + fact.copy(variable = callStatement.lhv.toPath()) ) } } @@ -387,7 +385,7 @@ class ForwardNpeFlowFunctions( add(TaintZeroFact) if (callStatement is JcAssignInst) { - val toPath = with(traits){ callStatement.lhv.toPath() } + val toPath = callStatement.lhv.toPath() val from = callStatement.rhv if (from is JcNullConstant || (from is JcCallExpr && from.method.method.isNullable == true)) { add(Tainted(toPath, TaintMark.NULLNESS)) @@ -401,11 +399,10 @@ class ForwardNpeFlowFunctions( if (config != null) { val conditionEvaluator = BasicConditionEvaluator( - traits, - CallPositionToValueResolver(traits, callStatement) + CallPositionToValueResolver(callStatement) ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(traits,callStatement) + CallPositionToAccessPathResolver(callStatement) ) // Handle MethodSource config items: @@ -437,12 +434,10 @@ class ForwardNpeFlowFunctions( } else { val facts = mutableSetOf() val conditionEvaluator = FactAwareConditionEvaluator( - traits, - CallPositionToValueResolver(traits,callStatement), - fact + fact, CallPositionToValueResolver(callStatement) ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(traits, callStatement) + CallPositionToAccessPathResolver(callStatement) ) var defaultBehavior = true @@ -494,7 +489,7 @@ class ForwardNpeFlowFunctions( } // FIXME: adhoc for constructors: - if ( with(traits) { callee.isConstructor }) { + if (callee.isConstructor) { return@FlowFunction listOf(fact) } @@ -515,14 +510,14 @@ class ForwardNpeFlowFunctions( for (actual in callExpr.args) { // Possibly tainted actual parameter: - if (fact.variable.startsWith( with(traits){ actual.toPathOrNull() })) { + if (fact.variable.startsWith(actual.toPathOrNull())) { return@FlowFunction emptyList() // Will be handled by summary edge } } if (callExpr is JcInstanceCallExpr) { // Possibly tainted instance: - if (fact.variable.startsWith( with(traits){ callExpr.instance.toPathOrNull() })) { + if (fact.variable.startsWith(callExpr.instance.toPathOrNull())) { return@FlowFunction emptyList() // Will be handled by summary edge } } @@ -531,7 +526,7 @@ class ForwardNpeFlowFunctions( if (callStatement is JcAssignInst) { // Possibly tainted lhv: - if (fact.variable.startsWith( with(traits){ callStatement.lhv.toPathOrNull() })) { + if (fact.variable.startsWith(callStatement.lhv.toPathOrNull())) { return@FlowFunction emptyList() // Overridden by rhv } } @@ -551,15 +546,13 @@ class ForwardNpeFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits) { - callStatement.getCallExpr() - ?: error("Call statement should have non-null callExpr") - } + val callExpr = callStatement.getCallExpr() + ?: error("Call statement should have non-null callExpr") buildSet { // Transmit facts on arguments (from 'actual' to 'formal'): val actualParams = callExpr.args - val formalParams = with(traits){ getArgumentsOf(callee) } + val formalParams = getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentActualToFormal( @@ -578,7 +571,7 @@ class ForwardNpeFlowFunctions( fact = fact, at = callStatement, from = callExpr.instance, - to = with(traits) { callee.thisInstance } + to = callee.thisInstance ) ) } @@ -604,7 +597,7 @@ class ForwardNpeFlowFunctions( // Note: returnValue can be null here in some weird cases, e.g. in lambda. exitStatement.returnValue?.let { returnValue -> if (returnValue is JcNullConstant) { - val toPath = with(traits) { callStatement.lhv.toPath() } + val toPath = callStatement.lhv.toPath() add(Tainted(toPath, TaintMark.NULLNESS)) } } @@ -613,7 +606,7 @@ class ForwardNpeFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits){ callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") val callee = graph.methodOf(exitStatement) @@ -621,7 +614,7 @@ class ForwardNpeFlowFunctions( // Transmit facts on arguments (from 'formal' back to 'actual'), if they are passed by-ref: if (fact.variable.isOnHeap) { val actualParams = callExpr.args - val formalParams = with(traits) { getArgumentsOf(callee) } + val formalParams = getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentFormalToActual( @@ -640,7 +633,7 @@ class ForwardNpeFlowFunctions( transmitTaintThisToInstance( fact = fact, at = callStatement, - from = with(traits){ callee.thisInstance }, + from = callee.thisInstance, to = callExpr.instance ) ) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt index 337ec1ec6..a9e52776d 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/npe/NpeManager.kt @@ -30,11 +30,11 @@ import org.jacodb.api.common.cfg.CommonInst private val logger = mu.KotlinLogging.logger {} +context(Traits) class NpeManager( - traits: Traits, graph: ApplicationGraph, unitResolver: UnitResolver, -) : TaintManager(traits, graph, unitResolver, useBidiRunner = false) +) : TaintManager(graph, unitResolver, useBidiRunner = false) where Method : CommonMethod, Statement : CommonInst { @@ -43,7 +43,7 @@ class NpeManager( ): TaintRunner { check(unit !in runnerForUnit) { "Runner for $unit already exists" } - val analyzer = NpeAnalyzer(traits, graph) + val analyzer = NpeAnalyzer(graph) val runner = UniRunner( graph = graph, analyzer = analyzer, diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt index 28f634d6a..675c9c2a1 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintAnalyzers.kt @@ -27,6 +27,7 @@ import org.jacodb.api.common.CommonMethod import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.api.common.cfg.CommonInst import org.jacodb.api.jvm.cfg.JcIfInst +import org.jacodb.impl.cfg.util.loops import org.jacodb.ets.base.EtsArrayAccess import org.jacodb.ets.base.EtsAssignStmt import org.jacodb.ets.base.EtsIfStmt @@ -34,7 +35,6 @@ import org.jacodb.ets.base.EtsNewArrayExpr import org.jacodb.ets.base.EtsStmt import org.jacodb.ets.graph.loops import org.jacodb.ets.utils.getOperands -import org.jacodb.impl.cfg.util.loops import org.jacodb.panda.staticvm.utils.loops import org.jacodb.taint.configuration.TaintConfigurationItem import org.jacodb.taint.configuration.TaintMethodSink @@ -42,8 +42,8 @@ import org.jacodb.panda.staticvm.cfg.PandaIfInst as StaticPandaIfInst private val logger = mu.KotlinLogging.logger {} +context(Traits) class TaintAnalyzer( - val traits: Traits, private val graph: ApplicationGraph, private val getConfigForMethod: (ForwardTaintFlowFunctions.(Method) -> List?)? = null, ) : Analyzer, Method, Statement> @@ -52,9 +52,9 @@ class TaintAnalyzer( override val flowFunctions: ForwardTaintFlowFunctions by lazy { if (getConfigForMethod != null) { - ForwardTaintFlowFunctions(traits, graph, getConfigForMethod) + ForwardTaintFlowFunctions(graph, getConfigForMethod) } else { - ForwardTaintFlowFunctions(traits, graph) + ForwardTaintFlowFunctions(graph) } } @@ -70,9 +70,9 @@ class TaintAnalyzer( } run { - val callExpr = with(traits) { edge.to.statement.getCallExpr() } ?: return@run + val callExpr = edge.to.statement.getCallExpr() ?: return@run - val callee = with(traits) { callExpr.callee } + val callee = callExpr.callee val config = with(flowFunctions) { getConfigForMethod(callee) } ?: return@run @@ -84,9 +84,8 @@ class TaintAnalyzer( // Determine whether 'edge.to' is a sink via config: val conditionEvaluator = FactAwareConditionEvaluator( - traits, - CallPositionToValueResolver(traits, edge.to.statement), edge.to.fact, + CallPositionToValueResolver(edge.to.statement), ) for (item in config.filterIsInstance()) { if (item.condition.accept(conditionEvaluator)) { @@ -153,7 +152,7 @@ class TaintAnalyzer( val expr = statement.rhv if (expr is EtsNewArrayExpr) { val arg = expr.size - if (with(traits) { arg.toPathOrNull() } == fact.variable) { + if (arg.toPathOrNull() == fact.variable) { val message = "Untrusted array size" val vulnerability = TaintVulnerability(message, sink = edge.to) add(NewVulnerability(vulnerability)) @@ -170,7 +169,7 @@ class TaintAnalyzer( for (op in statement.getOperands()) { if (op is EtsArrayAccess) { val arg = op.index - if (with(traits) { arg.toPathOrNull() } == fact.variable) { + if (arg.toPathOrNull() == fact.variable) { val message = "Untrusted index for access array" val vulnerability = TaintVulnerability(message, sink = edge.to) add(NewVulnerability(vulnerability)) @@ -190,15 +189,15 @@ class TaintAnalyzer( } } +context(Traits) class BackwardTaintAnalyzer( - val traits: Traits, private val graph: ApplicationGraph, ) : Analyzer, Method, Statement> where Method : CommonMethod, Statement : CommonInst { override val flowFunctions: BackwardTaintFlowFunctions by lazy { - BackwardTaintFlowFunctions(traits, graph) + BackwardTaintFlowFunctions(graph) } private fun isExitPoint(statement: Statement): Boolean { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt index b51d83ced..6a4cf1abf 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintFlowFunctions.kt @@ -73,8 +73,8 @@ import org.jacodb.panda.staticvm.cfg.PandaPhiInst as StaticPandaPhiInst private val logger = mu.KotlinLogging.logger {} // TODO: replace with CommonMethod, with CommonInst +context(Traits) class ForwardTaintFlowFunctions( - val traits: Traits, private val graph: ApplicationGraph, val getConfigForMethod: ForwardTaintFlowFunctions.(Method) -> List? = { method -> taintConfigurationFeature?.let { feature -> @@ -114,11 +114,10 @@ class ForwardTaintFlowFunctions( val config = getConfigForMethod(method) if (config != null) { val conditionEvaluator = BasicConditionEvaluator( - traits, - EntryPointPositionToValueResolver(traits, method) + EntryPointPositionToValueResolver(method) ) val actionEvaluator = TaintActionEvaluator( - EntryPointPositionToAccessPathResolver(traits, method) + EntryPointPositionToAccessPathResolver(method) ) // Handle EntryPointSource config items: @@ -141,8 +140,8 @@ class ForwardTaintFlowFunctions( from: CommonExpr, to: CommonValue, ): Collection { - val toPath = with(traits) { to.toPath() } - val fromPath = with(traits) { from.toPathOrNull() } + val toPath = to.toPath() + val fromPath = from.toPathOrNull() if (fromPath != null) { // Adhoc taint array: @@ -272,8 +271,8 @@ class ForwardTaintFlowFunctions( from: CommonValue, to: CommonValue, ): Collection = buildSet { - val fromPath = with(traits) { from.toPath() } - val toPath = with(traits) { to.toPath() } + val fromPath = from.toPath() + val toPath = to.toPath() val tail = (fact.variable - fromPath) ?: return@buildSet val newPath = toPath + tail @@ -315,10 +314,10 @@ class ForwardTaintFlowFunctions( callStatement: Statement, returnSite: Statement, // FIXME: unused? ) = FlowFunction { fact -> - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") - val callee = with(traits) { callExpr.callee } + val callee = callExpr.callee // FIXME: handle taint pass-through on invokedynamic-based String concatenation: if (fact is Tainted @@ -327,10 +326,10 @@ class ForwardTaintFlowFunctions( && callStatement is JcAssignInst ) { for (arg in callExpr.args) { - if (with(traits) { arg.toPath() } == fact.variable) { + if (arg.toPath() == fact.variable) { return@FlowFunction setOf( fact, - fact.copy(variable = with(traits) { callStatement.lhv.toPath() }) + fact.copy(variable = callStatement.lhv.toPath()) ) } } @@ -345,11 +344,10 @@ class ForwardTaintFlowFunctions( if (config != null) { val conditionEvaluator = BasicConditionEvaluator( - traits, - CallPositionToValueResolver(traits, callStatement) + CallPositionToValueResolver(callStatement) ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(traits, callStatement) + CallPositionToAccessPathResolver(callStatement) ) // Handle MethodSource config items: @@ -372,12 +370,10 @@ class ForwardTaintFlowFunctions( if (config != null) { val facts = mutableSetOf() val conditionEvaluator = FactAwareConditionEvaluator( - traits, - CallPositionToValueResolver(traits, callStatement), - fact, + fact, CallPositionToValueResolver(callStatement) ) val actionEvaluator = TaintActionEvaluator( - CallPositionToAccessPathResolver(traits, callStatement) + CallPositionToAccessPathResolver(callStatement) ) var defaultBehavior = true @@ -428,7 +424,7 @@ class ForwardTaintFlowFunctions( } // FIXME: adhoc for constructors: - if (with(traits) { callee.isConstructor }) { + if (callee.isConstructor) { return@FlowFunction listOf(fact) } @@ -449,14 +445,14 @@ class ForwardTaintFlowFunctions( for (actual in callExpr.args) { // Possibly tainted actual parameter: - if (fact.variable.startsWith(with(traits) { actual.toPathOrNull() })) { + if (fact.variable.startsWith(actual.toPathOrNull())) { return@FlowFunction emptyList() // Will be handled by summary edge } } if (callExpr is CommonInstanceCallExpr) { // Possibly tainted instance: - if (fact.variable.startsWith(with(traits) { callExpr.instance.toPathOrNull() })) { + if (fact.variable.startsWith(callExpr.instance.toPathOrNull())) { return@FlowFunction emptyList() // Will be handled by summary edge } } @@ -465,7 +461,7 @@ class ForwardTaintFlowFunctions( if (callStatement is CommonAssignInst) { // Possibly tainted lhv: - if (fact.variable.startsWith(with(traits) { callStatement.lhv.toPathOrNull() })) { + if (fact.variable.startsWith(callStatement.lhv.toPathOrNull())) { return@FlowFunction emptyList() // Overridden by rhv } } @@ -485,13 +481,13 @@ class ForwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") buildSet { // Transmit facts on arguments (from 'actual' to 'formal'): val actualParams = callExpr.args - val formalParams = traits.getArgumentsOf(callee) + val formalParams = getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll(transmitTaintArgumentActualToFormal(fact, from = actual, to = formal)) } @@ -502,7 +498,7 @@ class ForwardTaintFlowFunctions( transmitTaintInstanceToThis( fact = fact, from = callExpr.instance, - to = with(traits) { callee.thisInstance } + to = callee.thisInstance ) ) } @@ -524,7 +520,7 @@ class ForwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") val callee = graph.methodOf(exitStatement) @@ -532,7 +528,7 @@ class ForwardTaintFlowFunctions( // Transmit facts on arguments (from 'formal' back to 'actual'), if they are passed by-ref: if (fact.variable.isOnHeap) { val actualParams = callExpr.args - val formalParams = traits.getArgumentsOf(callee) + val formalParams = getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentFormalToActual( @@ -549,7 +545,7 @@ class ForwardTaintFlowFunctions( addAll( transmitTaintThisToInstance( fact = fact, - from = with(traits) { callee.thisInstance }, + from = callee.thisInstance, to = callExpr.instance ) ) @@ -571,8 +567,8 @@ class ForwardTaintFlowFunctions( } } +context(Traits) class BackwardTaintFlowFunctions( - val traits: Traits, private val graph: ApplicationGraph, ) : FlowFunctions where Method : CommonMethod, @@ -589,8 +585,8 @@ class BackwardTaintFlowFunctions( from: CommonValue, to: CommonExpr, ): Collection { - val fromPath = with(traits) { from.toPath() } - val toPath = with(traits) { to.toPathOrNull() } + val fromPath = from.toPath() + val toPath = to.toPathOrNull() if (toPath != null) { val tail = fact.variable - fromPath @@ -640,8 +636,8 @@ class BackwardTaintFlowFunctions( from: CommonValue, to: CommonValue, ): Collection = buildSet { - val fromPath = with(traits) { from.toPath() } - val toPath = with(traits) { to.toPath() } + val fromPath = from.toPath() + val toPath = to.toPath() val tail = (fact.variable - fromPath) ?: return@buildSet val newPath = toPath + tail @@ -690,9 +686,9 @@ class BackwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") - val callee = with(traits) { callExpr.callee } + val callee = callExpr.callee if (callee in graph.callees(callStatement)) { @@ -702,14 +698,14 @@ class BackwardTaintFlowFunctions( for (actual in callExpr.args) { // Possibly tainted actual parameter: - if (fact.variable.startsWith(with(traits) { actual.toPathOrNull() })) { + if (fact.variable.startsWith(actual.toPathOrNull())) { return@FlowFunction emptyList() // Will be handled by summary edge } } if (callExpr is CommonInstanceCallExpr) { // Possibly tainted instance: - if (fact.variable.startsWith(with(traits) { callExpr.instance.toPathOrNull() })) { + if (fact.variable.startsWith(callExpr.instance.toPathOrNull())) { return@FlowFunction emptyList() // Will be handled by summary edge } } @@ -718,7 +714,7 @@ class BackwardTaintFlowFunctions( if (callStatement is CommonAssignInst) { // Possibly tainted rhv: - if (fact.variable.startsWith(with(traits) { callStatement.rhv.toPathOrNull() })) { + if (fact.variable.startsWith(callStatement.rhv.toPathOrNull())) { return@FlowFunction emptyList() // Overridden by lhv } } @@ -738,13 +734,13 @@ class BackwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") buildSet { // Transmit facts on arguments (from 'actual' to 'formal'): val actualParams = callExpr.args - val formalParams = traits.getArgumentsOf(callee) + val formalParams = getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll(transmitTaintArgumentActualToFormal(fact, from = actual, to = formal)) } @@ -755,7 +751,7 @@ class BackwardTaintFlowFunctions( transmitTaintInstanceToThis( fact = fact, from = callExpr.instance, - to = with(traits) { callee.thisInstance } + to = callee.thisInstance ) ) } @@ -791,7 +787,7 @@ class BackwardTaintFlowFunctions( } check(fact is Tainted) - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") val callee = graph.methodOf(exitStatement) @@ -799,7 +795,7 @@ class BackwardTaintFlowFunctions( // Transmit facts on arguments (from 'formal' back to 'actual'), if they are passed by-ref: if (fact.variable.isOnHeap) { val actualParams = callExpr.args - val formalParams = traits.getArgumentsOf(callee) + val formalParams = getArgumentsOf(callee) for ((formal, actual) in formalParams.zip(actualParams)) { addAll( transmitTaintArgumentFormalToActual( @@ -816,7 +812,7 @@ class BackwardTaintFlowFunctions( addAll( transmitTaintThisToInstance( fact = fact, - from = with(traits) { callee.thisInstance }, + from = callee.thisInstance, to = callExpr.instance ) ) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt index 38baaa650..641092696 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/taint/TaintManager.kt @@ -51,12 +51,13 @@ import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds import kotlin.time.DurationUnit +import kotlin.time.ExperimentalTime import kotlin.time.TimeSource private val logger = mu.KotlinLogging.logger {} +context(Traits) open class TaintManager( - val traits: Traits, protected val graph: ApplicationGraph, protected val unitResolver: UnitResolver, private val useBidiRunner: Boolean = false, @@ -90,7 +91,7 @@ open class TaintManager( unitResolver = unitResolver, unit = unit, { manager -> - val analyzer = TaintAnalyzer(traits, graph, getConfigForMethod) + val analyzer = TaintAnalyzer(graph, getConfigForMethod) UniRunner( manager = manager, graph = graph, @@ -101,7 +102,7 @@ open class TaintManager( ) }, { manager -> - val analyzer = BackwardTaintAnalyzer(traits, graph) + val analyzer = BackwardTaintAnalyzer(graph) UniRunner( manager = manager, graph = graph.reversed, @@ -113,7 +114,7 @@ open class TaintManager( } ) } else { - val analyzer = TaintAnalyzer(traits, graph, getConfigForMethod) + val analyzer = TaintAnalyzer(graph, getConfigForMethod) UniRunner( manager = this@TaintManager, graph = graph, @@ -150,6 +151,7 @@ open class TaintManager( } @JvmName("analyze") // needed for Java interop because of inline class (Duration) + @OptIn(ExperimentalTime::class) fun analyze( startMethods: List, timeout: Duration = 3600.seconds, diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt index 0fdab9d21..148e04b9c 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableAnalyzer.kt @@ -24,15 +24,15 @@ import org.jacodb.api.common.CommonMethod import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.api.common.cfg.CommonInst +context(Traits) class UnusedVariableAnalyzer( - val traits: Traits, private val graph: ApplicationGraph, ) : Analyzer, Method, Statement> where Method : CommonMethod, Statement : CommonInst { override val flowFunctions: UnusedVariableFlowFunctions by lazy { - UnusedVariableFlowFunctions(traits, graph) + UnusedVariableFlowFunctions(graph) } private fun isExitPoint(statement: Statement): Boolean { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt index 0397a598d..2f3d9aa08 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableFlowFunctions.kt @@ -21,14 +21,15 @@ import org.jacodb.analysis.ifds.FlowFunctions import org.jacodb.analysis.ifds.isOnHeap import org.jacodb.analysis.util.Traits import org.jacodb.api.common.CommonMethod +import org.jacodb.api.common.CommonProject import org.jacodb.api.common.analysis.ApplicationGraph import org.jacodb.api.common.cfg.CommonAssignInst import org.jacodb.api.common.cfg.CommonInst import org.jacodb.api.jvm.cfg.JcSpecialCallExpr import org.jacodb.api.jvm.cfg.JcStaticCallExpr +context(Traits) class UnusedVariableFlowFunctions( - val traits: Traits, private val graph: ApplicationGraph, ) : FlowFunctions where Method : CommonMethod, @@ -49,7 +50,7 @@ class UnusedVariableFlowFunctions( } if (fact == UnusedVariableZeroFact) { - val toPath = with(traits) { current.lhv.toPath() } + val toPath = current.lhv.toPath() if (!toPath.isOnHeap) { return@FlowFunction setOf(UnusedVariableZeroFact, UnusedVariable(toPath, current)) } else { @@ -58,9 +59,9 @@ class UnusedVariableFlowFunctions( } check(fact is UnusedVariable) - val toPath = with(traits) { current.lhv.toPath() } + val toPath = current.lhv.toPath() val default = if (toPath == fact.variable) emptySet() else setOf(fact) - val fromPath = with(traits) { current.rhv.toPathOrNull() } + val fromPath = current.rhv.toPathOrNull() ?: return@FlowFunction default if (fromPath.isOnHeap || toPath.isOnHeap) { @@ -83,7 +84,7 @@ class UnusedVariableFlowFunctions( callStatement: Statement, calleeStart: Statement, ) = FlowFunction { fact -> - val callExpr = with(traits) { callStatement.getCallExpr() } + val callExpr = callStatement.getCallExpr() ?: error("Call statement should have non-null callExpr") if (fact == UnusedVariableZeroFact) { @@ -94,9 +95,9 @@ class UnusedVariableFlowFunctions( return@FlowFunction buildSet { add(UnusedVariableZeroFact) val callee = graph.methodOf(calleeStart) - val formalParams = traits.getArgumentsOf(callee) + val formalParams = getArgumentsOf(callee) for (formal in formalParams) { - add(UnusedVariable(with(traits) { formal.toPath() }, callStatement)) + add(UnusedVariable(formal.toPath(), callStatement)) } } } diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt index 1c3369148..687465415 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/unused/UnusedVariableManager.kt @@ -54,8 +54,8 @@ import kotlin.time.TimeSource private val logger = mu.KotlinLogging.logger {} +context(Traits) class UnusedVariableManager( - val traits: Traits, private val graph: ApplicationGraph, private val unitResolver: UnitResolver, ) : Manager, Method, Statement> @@ -79,7 +79,7 @@ class UnusedVariableManager( check(unit !in runnerForUnit) { "Runner for $unit already exists" } logger.debug { "Creating a new runner for $unit" } - val analyzer = UnusedVariableAnalyzer(traits, graph) + val analyzer = UnusedVariableAnalyzer(graph) val runner = UniRunner( graph = graph, analyzer = analyzer, @@ -198,7 +198,7 @@ class UnusedVariableManager( if (fact is UnusedVariable) { @Suppress("UNCHECKED_CAST") used.putIfAbsent(fact.initStatement as Statement, false) - if (with(traits) { fact.variable.isUsedAt(inst) }) { + if (fact.variable.isUsedAt(inst)) { used[fact.initStatement] = true } } diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt index 3601d92d0..489aeb37d 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/EtsTraits.kt @@ -26,12 +26,10 @@ import org.jacodb.api.common.cfg.CommonCallExpr import org.jacodb.api.common.cfg.CommonExpr import org.jacodb.api.common.cfg.CommonValue import org.jacodb.ets.base.CONSTRUCTOR_NAME -import org.jacodb.ets.base.EtsAnyType import org.jacodb.ets.base.EtsArrayAccess import org.jacodb.ets.base.EtsCallExpr import org.jacodb.ets.base.EtsCastExpr import org.jacodb.ets.base.EtsClassType -import org.jacodb.ets.base.EtsConstant import org.jacodb.ets.base.EtsEntity import org.jacodb.ets.base.EtsImmediate import org.jacodb.ets.base.EtsInstanceFieldRef @@ -50,12 +48,7 @@ import org.jacodb.analysis.util.toPathOrNull as _toPathOrNull import org.jacodb.ets.utils.getOperands as _getOperands import org.jacodb.ets.utils.getValues as _getValues -interface EtsTraits : Traits { - - companion object : EtsTraits { - // Note: unused for now - // lateinit var cp: EtsFile - } +class EtsTraits : Traits { override val CommonCallExpr.callee: EtsMethod get() { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt index 9f112cacd..fafd9e0bd 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/util/JcTraits.kt @@ -60,7 +60,7 @@ import org.jacodb.api.jvm.ext.cfg.callExpr as _callExpr * JVM-specific extensions for analysis. */ class JcTraits( - val cp: JcClasspath, + val cp: JcClasspath ) : Traits { override val JcMethod.thisInstance: JcThis diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt index edb7733f4..b14d902c8 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/ConditionEvaluatorTest.kt @@ -93,7 +93,9 @@ class ConditionEvaluatorTest { else -> null }.toMaybe() } - private val evaluator: ConditionVisitor = BasicConditionEvaluator(traits, positionResolver) + private val evaluator: ConditionVisitor = with(traits) { + BasicConditionEvaluator(positionResolver) + } @Test fun `True is true`() { @@ -326,9 +328,9 @@ class ConditionEvaluatorTest { } @Test - fun `FactAwareConditionEvaluator supports ContainsMark`() { - val fact = Tainted(with(traits) { intValue.toPath() }, TaintMark("FOO")) - val factAwareEvaluator = FactAwareConditionEvaluator(traits, positionResolver, fact) + fun `FactAwareConditionEvaluator supports ContainsMark`() = with(traits) { + val fact = Tainted(intValue.toPath(), TaintMark("FOO")) + val factAwareEvaluator = FactAwareConditionEvaluator(fact, positionResolver) assertTrue(factAwareEvaluator.visit(ContainsMark(intArg, TaintMark("FOO")))) assertFalse(factAwareEvaluator.visit(ContainsMark(intArg, TaintMark("BAR")))) assertFalse(factAwareEvaluator.visit(ContainsMark(stringArg, TaintMark("FOO")))) diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt index ea1e697a3..23097a333 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt @@ -199,7 +199,9 @@ abstract class IfdsNpeTest : BaseAnalysisTest() { private fun findSinks(method: JcMethod): List> { val unitResolver = SingletonUnitResolver - val manager = NpeManager(traits, graph, unitResolver) + val manager = with(traits) { + NpeManager(graph, unitResolver) + } return manager.analyze(listOf(method), timeout = 30.seconds) } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt index 4c7433a00..0b5812c88 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt @@ -62,7 +62,9 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { val method = cp.findClass().declaredMethods.single { it.name == methodName } val methods = listOf(method) val unitResolver = SingletonUnitResolver - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } val sinks = manager.analyze(methods, timeout = 30.seconds) assertTrue(sinks.isNotEmpty()) val sink = sinks.first() @@ -76,7 +78,9 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { fun `test on Juliet's CWE 89`(className: String) { testSingleJulietClass(className) { method -> val unitResolver = SingletonUnitResolver - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } manager.analyze(listOf(method), timeout = 30.seconds) } } @@ -86,7 +90,9 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { val className = "juliet.testcases.CWE89_SQL_Injection.s01.CWE89_SQL_Injection__connect_tcp_execute_01" testSingleJulietClass(className) { method -> val unitResolver = SingletonUnitResolver - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } manager.analyze(listOf(method), timeout = 30.seconds) } } @@ -97,7 +103,9 @@ abstract class IfdsSqlTest : BaseAnalysisTest() { val clazz = cp.findClass(className) val badMethod = clazz.methods.single { it.name == "bad" } val unitResolver = ClassUnitResolver(true) - val manager = TaintManager(traits, graph, unitResolver, useBidiRunner = true) + val manager = with(traits) { + TaintManager(graph, unitResolver, useBidiRunner = true) + } val sinks = manager.analyze(listOf(badMethod), timeout = 30.seconds) assertTrue(sinks.isNotEmpty()) val sink = sinks.first() diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt index 46a7c8880..4a35ebb8c 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsTaintTest.kt @@ -38,7 +38,9 @@ class IfdsTaintTest : BaseAnalysisTest() { private fun findSinks(method: JcMethod): List> { val unitResolver = SingletonUnitResolver - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } return manager.analyze(listOf(method), timeout = 3000.seconds) } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt index 93aeea395..f38aebcad 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUntrustedLoopBoundTest.kt @@ -59,7 +59,9 @@ class Ifds2UpperBoundTest : BaseAnalysisTest() { private inline fun testOneMethod(methodName: String) { val method = cp.findClass().declaredMethods.single { it.name == methodName } val unitResolver = SingletonUnitResolver - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } val sinks = manager.analyze(listOf(method), timeout = 60.seconds) logger.info { "Sinks: ${sinks.size}" } for (sink in sinks) { diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt index c6c9a32c0..3c4429b8a 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt @@ -63,7 +63,9 @@ abstract class IfdsUnusedTest : BaseAnalysisTest() { fun `test on Juliet's CWE 563`(className: String) { testSingleJulietClass(className) { method -> val unitResolver = SingletonUnitResolver - val manager = UnusedVariableManager(traits, graph, unitResolver) + val manager = with(traits) { + UnusedVariableManager(graph, unitResolver) + } manager.analyze(listOf(method), timeout = 30.seconds) } } @@ -75,19 +77,18 @@ abstract class IfdsUnusedTest : BaseAnalysisTest() { val clazz = cp.findClass(className) val badMethod = clazz.methods.single { it.name == "bad" } val unitResolver = SingletonUnitResolver - val manager = UnusedVariableManager(traits, graph, unitResolver) + val manager = with(traits) { + UnusedVariableManager(graph, unitResolver) + } val sinks = manager.analyze(listOf(badMethod), timeout = 30.seconds) Assertions.assertTrue(sinks.isNotEmpty()) } } - class IfdsUnusedSqlTest : IfdsUnusedTest() { - companion object : WithDB(Usages, InMemoryHierarchy) } class IfdsUnusedRAMTest : IfdsUnusedTest() { - companion object : WithRAMDB(Usages, InMemoryHierarchy) } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt index 9f57e8cdc..aabbb218a 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt @@ -16,16 +16,12 @@ package org.jacodb.analysis.impl -import kotlinx.coroutines.runBlocking import org.jacodb.analysis.ifds.SingletonUnitResolver import org.jacodb.analysis.npe.NpeManager import org.jacodb.analysis.taint.TaintManager import org.jacodb.analysis.unused.UnusedVariableManager -import org.jacodb.api.jvm.JcClasspath import org.jacodb.api.jvm.ext.findClass -import org.jacodb.taint.configuration.TaintConfigurationFeature import org.jacodb.testing.WithGlobalDB -import org.jacodb.testing.allClasspath import org.joda.time.DateTime import org.junit.jupiter.api.Test import kotlin.time.Duration.Companion.seconds @@ -36,24 +32,14 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { companion object : WithGlobalDB() - override val cp: JcClasspath = runBlocking { - val configFileName = "config_small.json" - val configResource = this.javaClass.getResourceAsStream("/$configFileName") - if (configResource != null) { - val configJson = configResource.bufferedReader().readText() - val configurationFeature = TaintConfigurationFeature.fromJson(configJson) - db.classpath(allClasspath, listOf(configurationFeature) + classpathFeatures) - } else { - super.cp - } - } - @Test fun `test taint analysis`() { val clazz = cp.findClass() val methods = clazz.declaredMethods val unitResolver = SingletonUnitResolver - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } val sinks = manager.analyze(methods, timeout = 60.seconds) logger.info { "Vulnerabilities found: ${sinks.size}" } } @@ -63,7 +49,9 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { val clazz = cp.findClass() val methods = clazz.declaredMethods val unitResolver = SingletonUnitResolver - val manager = NpeManager(traits, graph, unitResolver) + val manager = with(traits) { + NpeManager(graph, unitResolver) + } val sinks = manager.analyze(methods, timeout = 60.seconds) logger.info { "Vulnerabilities found: ${sinks.size}" } } @@ -73,7 +61,9 @@ class JodaDateTimeAnalysisTest : BaseAnalysisTest() { val clazz = cp.findClass() val methods = clazz.declaredMethods val unitResolver = SingletonUnitResolver - val manager = UnusedVariableManager(traits, graph, unitResolver) + val manager = with(traits) { + UnusedVariableManager(graph, unitResolver) + } val sinks = manager.analyze(methods, timeout = 60.seconds) logger.info { "Unused variables found: ${sinks.size}" } } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt index 6058b5cd9..ba18ceafe 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt @@ -53,7 +53,7 @@ open class TaintFlowFunctionsTest : BaseTest() { companion object : WithDB(Usages, InMemoryHierarchy) - final override val cp: JcClasspath = runBlocking { + override val cp: JcClasspath = runBlocking { val configFileName = "config_test.json" val configResource = this.javaClass.getResourceAsStream("/$configFileName") if (configResource != null) { @@ -65,12 +65,16 @@ open class TaintFlowFunctionsTest : BaseTest() { } } - private val traits = JcTraits(cp) + private val traits by lazy { + JcTraits(cp) + } private val graph: JcApplicationGraph = mockk { every { cp } returns this@TaintFlowFunctionsTest.cp every { callees(any()) } answers { - sequenceOf(with(traits) { arg(0).callExpr!!.callee }) + with(traits) { + sequenceOf(arg(0).callExpr!!.callee) + } } every { methodOf(any()) } answers { arg(0).location.method @@ -101,59 +105,59 @@ open class TaintFlowFunctionsTest : BaseTest() { } @Test - fun `test obtain start facts`() { - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + fun `test obtain start facts`() = with(traits) { + val flowSpace = ForwardTaintFlowFunctions(graph) val facts = flowSpace.obtainPossibleStartFacts(testMethod).toList() - val arg0 = traits.getArgument(testMethod.parameters[0])!! - val arg0Taint = Tainted(with(traits) { arg0.toPath() }, TaintMark("EXAMPLE")) + val arg0 = getArgument(testMethod.parameters[0])!! + val arg0Taint = Tainted(arg0.toPath(), TaintMark("EXAMPLE")) Assertions.assertEquals(listOf(TaintZeroFact, arg0Taint), facts) } @Test - fun `test sequential flow function assign mark`() { + fun `test sequential flow function assign mark`() = with(traits) { // "x := y", where 'y' is tainted, should result in both 'x' and 'y' to be tainted val x: JcLocal = JcLocalVar(1, "x", stringType) val y: JcLocal = JcLocalVar(2, "y", stringType) val inst = JcAssignInst(location = mockk(), lhv = x, rhv = y) - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + val flowSpace = ForwardTaintFlowFunctions(graph) val f = flowSpace.obtainSequentFlowFunction(inst, next = mockk()) - val yTaint = Tainted(with(traits) { y.toPath() }, TaintMark("TAINT")) - val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("TAINT")) + val yTaint = Tainted(y.toPath(), TaintMark("TAINT")) + val xTaint = Tainted(x.toPath(), TaintMark("TAINT")) val facts = f.compute(yTaint).toList() Assertions.assertEquals(listOf(yTaint, xTaint), facts) } @Test - fun `test call flow function assign mark`() { + fun `test call flow function assign mark`() = with(traits) { // "x := test(...)", where 'test' is a source, should result in 'x' to be tainted val x: JcLocal = JcLocalVar(1, "x", stringType) val callStatement = JcAssignInst(location = mockk(), lhv = x, rhv = mockk { every { method.method } returns testMethod }) - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + val flowSpace = ForwardTaintFlowFunctions(graph) val f = flowSpace.obtainCallToReturnSiteFlowFunction(callStatement, returnSite = mockk()) - val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("EXAMPLE")) + val xTaint = Tainted(x.toPath(), TaintMark("EXAMPLE")) val facts = f.compute(TaintZeroFact).toList() Assertions.assertEquals(listOf(TaintZeroFact, xTaint), facts) } @Test - fun `test call flow function remove mark`() { + fun `test call flow function remove mark`() = with(traits) { // "test(x)", where 'x' is tainted, should result in 'x' NOT to be tainted val x: JcLocal = JcLocalVar(1, "x", stringType) val callStatement = JcCallInst(location = mockk(), callExpr = mockk { every { method.method } returns testMethod every { args } returns listOf(x) }) - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + val flowSpace = ForwardTaintFlowFunctions(graph) val f = flowSpace.obtainCallToReturnSiteFlowFunction(callStatement, returnSite = mockk()) - val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("REMOVE")) + val xTaint = Tainted(x.toPath(), TaintMark("REMOVE")) val facts = f.compute(xTaint).toList() Assertions.assertTrue(facts.isEmpty()) } @Test - fun `test call flow function copy mark`() { + fun `test call flow function copy mark`() = with(traits) { // "y := test(x)" should result in 'y' to be tainted only when 'x' is tainted val x: JcLocal = JcLocalVar(1, "x", stringType) val y: JcLocal = JcLocalVar(2, "y", stringType) @@ -161,45 +165,45 @@ open class TaintFlowFunctionsTest : BaseTest() { every { method.method } returns testMethod every { args } returns listOf(x) }) - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + val flowSpace = ForwardTaintFlowFunctions(graph) val f = flowSpace.obtainCallToReturnSiteFlowFunction(callStatement, returnSite = mockk()) - val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("COPY")) - val yTaint = Tainted(with(traits) { y.toPath() }, TaintMark("COPY")) + val xTaint = Tainted(x.toPath(), TaintMark("COPY")) + val yTaint = Tainted(y.toPath(), TaintMark("COPY")) val facts = f.compute(xTaint).toList() Assertions.assertEquals(listOf(xTaint, yTaint), facts) // copy from x to y val other: JcLocal = JcLocalVar(10, "other", stringType) - val otherTaint = Tainted(with(traits) { other.toPath() }, TaintMark("OTHER")) + val otherTaint = Tainted(other.toPath(), TaintMark("OTHER")) val facts2 = f.compute(otherTaint).toList() Assertions.assertEquals(listOf(otherTaint), facts2) // pass-through } @Test - fun `test call to start flow function`() { + fun `test call to start flow function`() = with(traits) { // "test(x)", where 'x' is tainted, should result in 'x' (formal argument of 'test') to be tainted val x: JcLocal = JcLocalVar(1, "x", stringType) val callStatement = JcCallInst(location = mockk(), callExpr = mockk { every { method.method } returns testMethod every { args } returns listOf(x) }) - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + val flowSpace = ForwardTaintFlowFunctions(graph) val f = flowSpace.obtainCallToStartFlowFunction(callStatement, calleeStart = mockk { every { location } returns mockk { every { method } returns testMethod } }) - val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("TAINT")) - val arg0: JcArgument = traits.getArgument(testMethod.parameters[0])!! - val arg0Taint = Tainted(with(traits) { arg0.toPath() }, TaintMark("TAINT")) + val xTaint = Tainted(x.toPath(), TaintMark("TAINT")) + val arg0: JcArgument = getArgument(testMethod.parameters[0])!! + val arg0Taint = Tainted(arg0.toPath(), TaintMark("TAINT")) val facts = f.compute(xTaint).toList() Assertions.assertEquals(listOf(arg0Taint), facts) val other: JcLocal = JcLocalVar(10, "other", stringType) - val otherTaint = Tainted(with(traits) { other.toPath() }, TaintMark("TAINT")) + val otherTaint = Tainted(other.toPath(), TaintMark("TAINT")) val facts2 = f.compute(otherTaint).toList() Assertions.assertTrue(facts2.isEmpty()) } @Test - fun `test exit flow function`() { + fun `test exit flow function`() = with(traits) { // "x := test()" + "return y", where 'y' is tainted, should result in 'x' to be tainted val x: JcLocal = JcLocalVar(1, "x", stringType) val callStatement = JcAssignInst(location = mockk(), lhv = x, rhv = mockk { @@ -209,16 +213,15 @@ open class TaintFlowFunctionsTest : BaseTest() { val exitStatement = JcReturnInst(location = mockk { every { method } returns testMethod }, returnValue = y) - val flowSpace = ForwardTaintFlowFunctions(traits, graph) + val flowSpace = ForwardTaintFlowFunctions(graph) val f = flowSpace.obtainExitToReturnSiteFlowFunction(callStatement, returnSite = mockk(), exitStatement) - val yTaint = Tainted(with(traits) { y.toPath() }, TaintMark("TAINT")) - val xTaint = Tainted(with(traits) { x.toPath() }, TaintMark("TAINT")) + val yTaint = Tainted(y.toPath(), TaintMark("TAINT")) + val xTaint = Tainted(x.toPath(), TaintMark("TAINT")) val facts = f.compute(yTaint).toList() Assertions.assertEquals(listOf(xTaint), facts) } } class TaintFlowFunctionsRAMTest : TaintFlowFunctionsTest() { - companion object : WithRAMDB(Usages, InMemoryHierarchy) } diff --git a/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt b/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt index 55f5fbe0e..26c6e7b10 100644 --- a/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt +++ b/jacodb-cli/src/main/kotlin/org/jacodb/cli/main.kt @@ -62,28 +62,35 @@ data class AnalysisConfig(val analyses: Map) fun launchAnalysesByConfig( config: AnalysisConfig, - traits: JcTraits, graph: JcApplicationGraph, methods: List, ): List>> { return config.analyses.mapNotNull { (analysis, options) -> + val traits = JcTraits(graph.cp) + val unitResolver = options["UnitResolver"]?.let { UnitResolver.getByName(it) } ?: SingletonUnitResolver when (analysis) { "NPE" -> { - val manager = NpeManager(traits, graph, unitResolver) + val manager = with(traits) { + NpeManager(graph, unitResolver) + } manager.analyze(methods, timeout = 60.seconds).map { it.toSarif(manager.vulnerabilityTraceGraph(it)) } } "Unused" -> { - val manager = UnusedVariableManager(traits, graph, unitResolver) + val manager = with(traits) { + UnusedVariableManager(graph, unitResolver) + } manager.analyze(methods, timeout = 60.seconds).map { it.toSarif() } } "SQL" -> { - val manager = TaintManager(traits, graph, unitResolver) + val manager = with(traits) { + TaintManager(graph, unitResolver) + } manager.analyze(methods, timeout = 60.seconds).map { it.toSarif(manager.vulnerabilityTraceGraph(it)) } } @@ -169,12 +176,11 @@ fun main(args: Array) { }).get() val startJcMethods = startJcClasses.flatMap { it.declaredMethods }.filter { !it.isPrivate } - val traits = JcTraits(cp) val graph = runBlocking { cp.newApplicationGraphForAnalysis() } - val vulnerabilities = launchAnalysesByConfig(config, traits, graph, startJcMethods).flatten() + val vulnerabilities = launchAnalysesByConfig(config, graph, startJcMethods).flatten() val report = sarifReportFromVulnerabilities(vulnerabilities) val prettyJson = Json { prettyPrint = true diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt index e7ad90720..067931a14 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt @@ -54,7 +54,7 @@ private val logger = mu.KotlinLogging.logger {} class EtsIfds { companion object { - private val traits = EtsTraits + private val traits = EtsTraits() private const val BASE_PATH = "/etsir/samples" @@ -97,12 +97,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } val methods = project.classes.flatMap { it.methods }.filter { it.name == "main" } logger.info { "Methods: ${methods.size}" } @@ -143,12 +144,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } val methods = project.classes.flatMap { it.methods } logger.info { "Methods: ${methods.size}" } @@ -208,12 +210,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } val goodMethod = project.classes.flatMap { it.methods }.single { it.name == "good" } logger.info { "good() method: $goodMethod" } @@ -249,12 +252,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } TaintAnalysisOptions.UNTRUSTED_LOOP_BOUND_SINK = true val methods = project.classes.flatMap { it.methods } @@ -288,12 +292,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } TaintAnalysisOptions.UNTRUSTED_ARRAY_SIZE_SINK = true val methods = project.classes.flatMap { it.methods } @@ -335,12 +340,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } val methods = project.classes.flatMap { it.methods } logger.info { "Methods: ${methods.size}" } @@ -415,12 +421,13 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } val methodNames = setOf( "getDeviceIdListWithCursor", @@ -438,5 +445,4 @@ class EtsIfds { logger.info { "Sinks: $sinks" } Assertions.assertTrue(sinks.isNotEmpty()) } - } diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt index 2159a0875..18378172f 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt @@ -51,9 +51,10 @@ class EtsProjectAnalysis { private var totalSinks: MutableList> = mutableListOf() companion object { - private val traits = EtsTraits + private val traits = EtsTraits() - private const val SOURCE_PROJECT_PATH = "/projects/applications_app_samples/source/applications_app_samples/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc" + private const val SOURCE_PROJECT_PATH = + "/projects/applications_app_samples/source/applications_app_samples/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc" private const val PROJECT_PATH = "/projects/applications_app_samples/etsir/ast/ArkTSDistributedCalc" private const val START_PATH = "/entry/src/main/ets" private const val BASE_PATH = PROJECT_PATH @@ -141,12 +142,13 @@ class EtsProjectAnalysis { private fun runAnalysis(project: EtsScene) { val graph = EtsApplicationGraphImpl(project) val unitResolver = UnitResolver { SingletonUnit } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = { method -> getConfigForMethod(method, rules) }, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = { method -> getConfigForMethod(method, rules) }, + ) + } val methods = project.classes.flatMap { it.methods } val sinks = manager.analyze(methods, timeout = 10.seconds) totalPathEdges += manager.runnerForUnit.values.sumOf { it.getPathEdges().size } diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt index 556553958..f98723677 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt @@ -49,10 +49,9 @@ private val logger = mu.KotlinLogging.logger {} class EtsTaintAnalysisTest { companion object { - private val traits = EtsTraits + private val traits = EtsTraits() private const val BASE_PATH = "/samples/etsir/ast" - private const val DECOMPILED_PATH = "/decompiled" private fun loadFromProject(name: String): EtsFile { @@ -110,12 +109,13 @@ class EtsTaintAnalysisTest { val graph = EtsApplicationGraphImpl(project) val unitResolver = UnitResolver { SingletonUnit } - val manager = TaintManager( - traits = traits, - graph = graph, - unitResolver = unitResolver, - getConfigForMethod = getConfigForMethod, - ) + val manager = with(traits) { + TaintManager( + graph = graph, + unitResolver = unitResolver, + getConfigForMethod = getConfigForMethod, + ) + } val methods = project.classes.flatMap { it.methods }.filter { it.name == "bad" } logger.info { "Methods: ${methods.size}" } diff --git a/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt b/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt index 7bb71c3c2..e4bb096b5 100644 --- a/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt +++ b/jacodb-panda-static/src/test/kotlin/panda/PandaDemoCases.kt @@ -44,9 +44,7 @@ private val logger = mu.KotlinLogging.logger {} class PandaDemoCases { - companion object { - private val traits = PandaStaticTraits - } + companion object : PandaStaticTraits private fun loadProject(path: String): PandaProject { val program = loadProgram("/$path") @@ -80,7 +78,6 @@ class PandaDemoCases { rules.ifEmpty { null } } val manager = TaintManager( - traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, @@ -115,7 +112,7 @@ class PandaDemoCases { ), ) ) - if (with(traits) { method.isConstructor } && method.enclosingClass.name == "escompat.ArrayBuffer") add( + if (method.isConstructor && method.enclosingClass.name == "escompat.ArrayBuffer") add( TaintMethodSink( method = method, ruleNote = "ArrayBuffer constructor", @@ -128,7 +125,6 @@ class PandaDemoCases { rules.ifEmpty { null } } val manager = TaintManager( - traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, diff --git a/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt b/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt index f612df1bc..fc358d299 100644 --- a/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt +++ b/jacodb-panda-static/src/test/kotlin/panda/PandaIfdsTest.kt @@ -46,9 +46,7 @@ private val logger = mu.KotlinLogging.logger {} class PandaIfdsTest { - companion object { - private val traits = PandaStaticTraits - } + companion object : PandaStaticTraits private fun stdlibAvailable() = EtsStdlib.stdlibAvailable() @@ -116,7 +114,6 @@ class PandaIfdsTest { rules.ifEmpty { null } } val manager = TaintManager( - traits = traits, graph = graph, unitResolver = unitResolver, getConfigForMethod = getConfigForMethod, From edfa8936e6aed12173d15a4b84126ad597897a84 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 16:23:00 +0300 Subject: [PATCH 109/120] Bump Kotlin to 2.1.0 --- buildSrc/src/main/kotlin/Dependencies.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index 732c6c7e5..1ac3c0864 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -22,7 +22,7 @@ object Versions { const val jooq = "3.14.16" const val juliet = "1.3.2" const val junit = "5.9.2" - const val kotlin = "2.0.20" + const val kotlin = "2.1.0" const val kotlin_logging = "1.8.3" const val kotlinx_benchmark = "0.4.6" const val kotlinx_cli = "0.3.5" From 5786147e640b5a1b2b53ffa195fb2419ac653e2b Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 17:28:37 +0300 Subject: [PATCH 110/120] Add EtsAliasType and support its se-/de-serialization --- .../kotlin/org/jacodb/ets/base/EtsType.kt | 56 +++++++++++++++++++ .../main/kotlin/org/jacodb/ets/dto/Convert.kt | 45 ++++++++++----- .../org/jacodb/ets/model/EtsSignature.kt | 9 +++ 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt index 3c059ff91..22c380522 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/base/EtsType.kt @@ -19,7 +19,9 @@ package org.jacodb.ets.base import org.jacodb.api.common.CommonType import org.jacodb.api.common.CommonTypeName import org.jacodb.ets.model.EtsClassSignature +import org.jacodb.ets.model.EtsLocalSignature import org.jacodb.ets.model.EtsMethodSignature +import org.jacodb.ets.model.EtsNamespaceSignature interface EtsType : CommonType, CommonTypeName { override val typeName: String @@ -46,6 +48,9 @@ interface EtsType : CommonType, CommonTypeName { fun visit(type: EtsArrayObjectType): R fun visit(type: EtsUnclearRefType): R fun visit(type: EtsGenericType): R + fun visit(type: EtsAliasType): R + fun visit(type: EtsAnnotationNamespaceType): R + fun visit(type: EtsAnnotationTypeQueryType): R interface Default : Visitor { override fun visit(type: EtsAnyType): R = defaultVisit(type) @@ -66,6 +71,9 @@ interface EtsType : CommonType, CommonTypeName { override fun visit(type: EtsArrayObjectType): R = defaultVisit(type) override fun visit(type: EtsUnclearRefType): R = defaultVisit(type) override fun visit(type: EtsGenericType): R = defaultVisit(type) + override fun visit(type: EtsAliasType): R = defaultVisit(type) + override fun visit(type: EtsAnnotationNamespaceType): R = defaultVisit(type) + override fun visit(type: EtsAnnotationTypeQueryType): R = defaultVisit(type) fun defaultVisit(type: EtsType): R } @@ -316,3 +324,51 @@ data class EtsGenericType( return visitor.visit(this) } } + +data class EtsAliasType( + val name: String, + val originalType: EtsType, + val signature: EtsLocalSignature, +) : EtsType { + override val typeName: String + get() = name + + override fun toString(): String { + return "$name = $originalType" + } + + override fun accept(visitor: EtsType.Visitor): R { + return visitor.visit(this) + } +} + +data class EtsAnnotationNamespaceType( + val originType: String, + val namespaceSignature: EtsNamespaceSignature, +) : EtsType { + override val typeName: String + get() = originType + + override fun toString(): String { + return originType + } + + override fun accept(visitor: EtsType.Visitor): R { + return visitor.visit(this) + } +} + +data class EtsAnnotationTypeQueryType( + val originType: String, +) : EtsType { + override val typeName: String + get() = originType + + override fun toString(): String { + return originType + } + + override fun accept(visitor: EtsType.Visitor): R { + return visitor.visit(this) + } +} diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt index 38fd338fc..81605cd7b 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/dto/Convert.kt @@ -18,7 +18,10 @@ package org.jacodb.ets.dto import org.jacodb.ets.base.CONSTRUCTOR_NAME import org.jacodb.ets.base.EtsAddExpr +import org.jacodb.ets.base.EtsAliasType import org.jacodb.ets.base.EtsAndExpr +import org.jacodb.ets.base.EtsAnnotationNamespaceType +import org.jacodb.ets.base.EtsAnnotationTypeQueryType import org.jacodb.ets.base.EtsAnyType import org.jacodb.ets.base.EtsArrayAccess import org.jacodb.ets.base.EtsArrayLiteral @@ -119,6 +122,7 @@ import org.jacodb.ets.model.EtsFieldSignature import org.jacodb.ets.model.EtsFieldSubSignature import org.jacodb.ets.model.EtsFile import org.jacodb.ets.model.EtsFileSignature +import org.jacodb.ets.model.EtsLocalSignature import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsMethodImpl import org.jacodb.ets.model.EtsMethodParameter @@ -615,9 +619,21 @@ fun convertToEtsType(type: TypeDto): EtsType { } } - is AliasTypeDto -> EtsUnknownType // TODO: EtsAliasType - is AnnotationNamespaceTypeDto -> EtsUnknownType // TODO: EtsAnnotationNamespaceType - is AnnotationTypeQueryTypeDto -> EtsUnknownType // TODO: EtsAnnotationTypeQueryType + is AliasTypeDto -> EtsAliasType( + name = type.name, + originalType = convertToEtsType(type.originalType), + signature = convertToEtsLocalSignature(type.signature), + ) + + is AnnotationNamespaceTypeDto -> EtsAnnotationNamespaceType( + originType = type.originType, + namespaceSignature = convertToEtsNamespaceSignature(type.namespaceSignature), + ) + + + is AnnotationTypeQueryTypeDto -> EtsAnnotationTypeQueryType( + originType = type.originType, + ) AnyTypeDto -> EtsAnyType @@ -638,18 +654,14 @@ fun convertToEtsType(type: TypeDto): EtsType { typeParameters = type.typeParameters.map { convertToEtsType(it) }, ) - is GenericTypeDto -> { - val defaultType = type.defaultType?.let { convertToEtsType(it) } - val constraint = type.constraint?.let { convertToEtsType(it) } - EtsGenericType( - name = type.name, - defaultType = defaultType, - constraint = constraint, - ) - } + is GenericTypeDto -> EtsGenericType( + name = type.name, + defaultType = type.defaultType?.let { convertToEtsType(it) }, + constraint = type.constraint?.let { convertToEtsType(it) }, + ) is LiteralTypeDto -> EtsLiteralType( - literalTypeName = type.literal.toString() + literalTypeName = type.literal.toString(), ) NeverTypeDto -> EtsNeverType @@ -764,6 +776,13 @@ fun convertToEtsMethodSignature(method: MethodSignatureDto): EtsMethodSignature ) } +fun convertToEtsLocalSignature(local: LocalSignatureDto): EtsLocalSignature { + return EtsLocalSignature( + name = local.name, + method = convertToEtsMethodSignature(local.method), + ) +} + fun convertToEtsMethod(method: MethodDto): EtsMethod { val signature = convertToEtsMethodSignature(method.signature) val typeParameters = method.typeParameters?.map { convertToEtsType(it) } ?: emptyList() diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt index a96cc460d..28f8bba54 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsSignature.kt @@ -125,3 +125,12 @@ data class EtsMethodParameter( return "$name${if (isOptional) "?" else ""}: $type" } } + +data class EtsLocalSignature( + val name: String, + val method: EtsMethodSignature, +) { + override fun toString(): String { + return "${method}#$name" + } +} From 1167b3541526f2c3bf9275431d1af8b87d8bcd7c Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 17:31:48 +0300 Subject: [PATCH 111/120] Make percent Double --- .../kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index fed2fdddb..d198829ec 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -30,6 +30,7 @@ import org.jacodb.ets.model.EtsScene import org.jacodb.ets.utils.callExpr import org.jacodb.impl.util.Maybe import java.util.concurrent.atomic.AtomicInteger +import kotlin.math.roundToInt private val logger = KotlinLogging.logger {} @@ -169,16 +170,16 @@ class EtsApplicationGraphImpl( if (total == 0) { return "[N/A%] $description ${property.get()}" } - val percent = 100 * property.get() / total - return "[$percent%] $description: ${property.get()}" + val percent = 100.0 * property.get() / total + return "[${percent.roundToInt()}%] $description: ${property.get()}" } private fun show(description: String, property: Int): String { if (total == 0) { return "[N/A%] $description $property" } - val percent = 100 * property / total - return "[$percent%] $description: $property" + val percent = 100.0 * property / total + return "[${percent.roundToInt()}%] $description: $property" } override fun toString(): String { From 597ce9c3b3a1d9903dbc157904174cb118cc2a3d Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 17:33:03 +0300 Subject: [PATCH 112/120] Remove unnecessary file signature comparison --- .../src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index d198829ec..75aab837a 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -230,7 +230,7 @@ class EtsApplicationGraphImpl( val matched = cp.classes .asSequence() - .filter { it.signature == signature && it.signature.file == signature.file } + .filter { it.signature == signature } .toList() if (matched.isEmpty()) { cacheClassWithIdealSignature[signature] = Maybe.none() From ed6634801cdbe37ee412983113abeb693d674f03 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 17:38:07 +0300 Subject: [PATCH 113/120] Add missing brackets to if-statement --- .../main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 75aab837a..ef8996f86 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -338,10 +338,11 @@ class EtsApplicationGraphImpl( // NOTE: cache lookup MUST be performed AFTER trying to match the neighbour! if (callee in cachePartiallyMatchedCallees) { val s = cachePartiallyMatchedCallees.getValue(callee).asSequence() - if (s.none()) + if (s.none()) { stats.cachedPartialMatchAsUnknown.incrementAndGet() - else + } else { stats.resolvedByPartiallyMatchedCache.incrementAndGet() + } return s } From 8631e6f17e5be23401d7c2f983f6059d74fe9ae9 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 29 Nov 2024 17:38:24 +0300 Subject: [PATCH 114/120] Add trailing comma --- .../src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index ef8996f86..18a2b01f8 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -109,7 +109,7 @@ class EtsApplicationGraphImpl( val cachedPartialMatchAsUnknown: AtomicInteger = AtomicInteger(0), val noPartialMatch: AtomicInteger = AtomicInteger(0), val multiplePartialMatch: AtomicInteger = AtomicInteger(0), - val partialMatchFound: AtomicInteger = AtomicInteger(0) + val partialMatchFound: AtomicInteger = AtomicInteger(0), ) { val total: Int get() = listOf( From 7952a37f47ab5876f719b11a84565d959068ca51 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Wed, 4 Dec 2024 12:55:18 +0300 Subject: [PATCH 115/120] Bump AA --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f2d4e5b3b..3ce838ddc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -64,7 +64,7 @@ jobs: DEST_DIR="arkanalyzer" MAX_RETRIES=10 RETRY_DELAY=3 # Delay between retries in seconds - BRANCH="neo/2024-11-21" + BRANCH="neo/2024-12-04" for ((i=1; i<=MAX_RETRIES; i++)); do git clone --depth=1 --branch $BRANCH $REPO_URL $DEST_DIR && break From 4e9bf44f2e211c52659c572bcbe5982f986f87ea Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Fri, 6 Dec 2024 18:31:35 +0300 Subject: [PATCH 116/120] Inline traits --- .../test/kotlin/org/jacodb/ets/test/EtsIfds.kt | 16 +++++++--------- .../org/jacodb/ets/test/EtsProjectAnalysis.kt | 4 +--- .../org/jacodb/ets/test/EtsTaintAnalysisTest.kt | 4 +--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt index 067931a14..05fdb1a8d 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsIfds.kt @@ -54,8 +54,6 @@ private val logger = mu.KotlinLogging.logger {} class EtsIfds { companion object { - private val traits = EtsTraits() - private const val BASE_PATH = "/etsir/samples" private fun loadSample(programName: String): EtsFile { @@ -97,7 +95,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, @@ -144,7 +142,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, @@ -210,7 +208,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, @@ -252,7 +250,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, @@ -292,7 +290,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, @@ -340,7 +338,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, @@ -421,7 +419,7 @@ class EtsIfds { } rules.ifEmpty { null } } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt index 18378172f..369bebafb 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsProjectAnalysis.kt @@ -51,8 +51,6 @@ class EtsProjectAnalysis { private var totalSinks: MutableList> = mutableListOf() companion object { - private val traits = EtsTraits() - private const val SOURCE_PROJECT_PATH = "/projects/applications_app_samples/source/applications_app_samples/code/SuperFeature/DistributedAppDev/ArkTSDistributedCalc" private const val PROJECT_PATH = "/projects/applications_app_samples/etsir/ast/ArkTSDistributedCalc" @@ -142,7 +140,7 @@ class EtsProjectAnalysis { private fun runAnalysis(project: EtsScene) { val graph = EtsApplicationGraphImpl(project) val unitResolver = UnitResolver { SingletonUnit } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt index f98723677..93aa05194 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsTaintAnalysisTest.kt @@ -49,8 +49,6 @@ private val logger = mu.KotlinLogging.logger {} class EtsTaintAnalysisTest { companion object { - private val traits = EtsTraits() - private const val BASE_PATH = "/samples/etsir/ast" private const val DECOMPILED_PATH = "/decompiled" @@ -109,7 +107,7 @@ class EtsTaintAnalysisTest { val graph = EtsApplicationGraphImpl(project) val unitResolver = UnitResolver { SingletonUnit } - val manager = with(traits) { + val manager = with(EtsTraits()) { TaintManager( graph = graph, unitResolver = unitResolver, From a351bf46c2a1c152814a3e4f4b8c39a4404c16f8 Mon Sep 17 00:00:00 2001 From: Konstantin Chukharev Date: Mon, 9 Dec 2024 16:21:14 +0300 Subject: [PATCH 117/120] Move Maybe to jacodb-ets --- .../src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt | 2 +- .../src/main/kotlin/org/jacodb/ets/utils}/Maybe.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename {jacodb-core/src/main/kotlin/org/jacodb/impl/util => jacodb-ets/src/main/kotlin/org/jacodb/ets/utils}/Maybe.kt (98%) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 18a2b01f8..d16edb4ee 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -28,7 +28,7 @@ import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.model.EtsScene import org.jacodb.ets.utils.callExpr -import org.jacodb.impl.util.Maybe +import org.jacodb.ets.utils.Maybe import java.util.concurrent.atomic.AtomicInteger import kotlin.math.roundToInt diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/util/Maybe.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Maybe.kt similarity index 98% rename from jacodb-core/src/main/kotlin/org/jacodb/impl/util/Maybe.kt rename to jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Maybe.kt index 808d2c590..b27a89db6 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/util/Maybe.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/Maybe.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.jacodb.impl.util +package org.jacodb.ets.utils @JvmInline value class Maybe private constructor( From 473c864eca05beba5fcbb020a80cb37f17affef6 Mon Sep 17 00:00:00 2001 From: Alexey Menshutin Date: Wed, 25 Dec 2024 12:34:07 +0300 Subject: [PATCH 118/120] Separate sdk files from the project ones in EtsScene --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 6 +- .../kotlin/org/jacodb/ets/model/EtsScene.kt | 13 ++++- .../org/jacodb/ets/utils/LoadEtsFile.kt | 57 +++++++++++++------ .../org/jacodb/ets/test/utils/LoadEts.kt | 2 +- 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index d16edb4ee..9a895d01b 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -27,8 +27,8 @@ import org.jacodb.ets.model.EtsFileSignature import org.jacodb.ets.model.EtsMethod import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.model.EtsScene -import org.jacodb.ets.utils.callExpr import org.jacodb.ets.utils.Maybe +import org.jacodb.ets.utils.callExpr import java.util.concurrent.atomic.AtomicInteger import kotlin.math.roundToInt @@ -228,7 +228,7 @@ class EtsApplicationGraphImpl( return cacheClassWithIdealSignature.getValue(signature) } - val matched = cp.classes + val matched = cp.projectAndSdkClasses .asSequence() .filter { it.signature == signature } .toList() @@ -348,7 +348,7 @@ class EtsApplicationGraphImpl( // If the neighbour match failed, // try to *uniquely* resolve the callee via a partial signature match: - val resolved = cp.classes + val resolved = cp.projectAndSdkClasses .asSequence() .filter { compareClassSignatures(it.signature, callee.enclosingClass) != ComparisonResult.NotEqual } // Note: exclude current class: diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt index 179473fc1..0f593308d 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/model/EtsScene.kt @@ -19,8 +19,15 @@ package org.jacodb.ets.model import org.jacodb.api.common.CommonProject class EtsScene( - val files: List, + val projectFiles: List, + val sdkFiles: List = emptyList(), ) : CommonProject { - val classes: List - get() = files.flatMap { it.allClasses } + val projectClasses: List + get() = projectFiles.flatMap { it.allClasses } + + val sdkClasses: List + get() = sdkFiles.flatMap { it.allClasses } + + val projectAndSdkClasses: List + get() = projectClasses + sdkClasses } diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt index 2a33ad796..695b2ed1e 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/utils/LoadEtsFile.kt @@ -43,7 +43,7 @@ private const val ENV_VAR_NODE_EXECUTABLE = "NODE_EXECUTABLE" private const val DEFAULT_NODE_EXECUTABLE = "node" fun generateEtsIR( - path: Path, + projectPath: Path, isProject: Boolean = false, loadEntrypoints: Boolean = true, useArkAnalyzerTypeInference: Int? = null, @@ -69,9 +69,9 @@ fun generateEtsIR( val node = System.getenv(ENV_VAR_NODE_EXECUTABLE) ?: DEFAULT_NODE_EXECUTABLE val output = if (isProject) { - createTempDirectory(prefix = path.nameWithoutExtension) + createTempDirectory(prefix = projectPath.nameWithoutExtension) } else { - kotlin.io.path.createTempFile(prefix = path.nameWithoutExtension, suffix = ".json") + kotlin.io.path.createTempFile(prefix = projectPath.nameWithoutExtension, suffix = ".json") } val cmd = listOfNotNull( @@ -80,16 +80,23 @@ fun generateEtsIR( if (isProject) "-p" else null, if (loadEntrypoints) "-e" else null, useArkAnalyzerTypeInference?.let { "-t $it" }, - path.pathString, + projectPath.pathString, output.pathString, ) runProcess(cmd, 10.seconds) return output } -fun loadEtsFileAutoConvert(path: Path): EtsFile { +fun generateSdkIR(sdkPath: Path): Path = generateEtsIR( + sdkPath, + isProject = true, + loadEntrypoints = false, + useArkAnalyzerTypeInference = 0, +) + +fun loadEtsFileAutoConvert(projectPath: Path): EtsFile { val irFilePath = generateEtsIR( - path, + projectPath, isProject = false, useArkAnalyzerTypeInference = 1, ) @@ -100,25 +107,39 @@ fun loadEtsFileAutoConvert(path: Path): EtsFile { } fun loadEtsProjectAutoConvert( - path: Path, + projectPath: Path, + sdkIRPath: Path? = null, loadEntrypoints: Boolean = false, useArkAnalyzerTypeInference: Int? = 1, ): EtsScene { val irFolderPath = generateEtsIR( - path, + projectPath, isProject = true, loadEntrypoints = loadEntrypoints, useArkAnalyzerTypeInference = useArkAnalyzerTypeInference, ) - val files = irFolderPath - .walk() - .filter { it.extension == "json" } - .map { - it.inputStream().use { stream -> - val etsFileDto = EtsFileDto.loadFromJson(stream) - convertToEtsFile(etsFileDto) + + return loadEtsProjectFromIR(irFolderPath, sdkIRPath) +} + +fun loadEtsProjectFromIR( + projectFilesPath: Path, + sdkFilesPath: Path?, +): EtsScene { + val walker = { irFolder: Path -> + irFolder.walk() + .filter { it.extension == "json" } + .map { + it.inputStream().use { stream -> + val etsFileDto = EtsFileDto.loadFromJson(stream) + convertToEtsFile(etsFileDto) + } } - } - .toList() - return EtsScene(files) + .toList() + } + + val projectFiles = walker(projectFilesPath) + val sdkFiles = sdkFilesPath?.let { walker(it) }.orEmpty() + + return EtsScene(projectFiles, sdkFiles) } diff --git a/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/LoadEts.kt b/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/LoadEts.kt index e62bc8e87..2249870cb 100644 --- a/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/LoadEts.kt +++ b/jacodb-ets/src/testFixtures/kotlin/org/jacodb/ets/test/utils/LoadEts.kt @@ -89,7 +89,7 @@ fun loadEtsProjectFromResources( val dirPaths = modules.map { "$prefix/$it" } val files = loadMultipleEtsFilesFromMultipleResourceDirectories(dirPaths).toList() logger.info { "Loaded ${files.size} files" } - return EtsScene(files) + return EtsScene(files, sdkFiles = emptyList()) } //----------------------------------------------------------------------------- From 9396eaa9308210f60fcbc59e7f361c4b6057d978 Mon Sep 17 00:00:00 2001 From: Alexey Menshutin Date: Wed, 25 Dec 2024 12:44:58 +0300 Subject: [PATCH 119/120] Remove statistics in callee resolver --- .../jacodb/ets/graph/EtsApplicationGraph.kt | 155 +----------------- 1 file changed, 3 insertions(+), 152 deletions(-) diff --git a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt index 9a895d01b..071fe78bb 100644 --- a/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt +++ b/jacodb-ets/src/main/kotlin/org/jacodb/ets/graph/EtsApplicationGraph.kt @@ -29,8 +29,6 @@ import org.jacodb.ets.model.EtsMethodSignature import org.jacodb.ets.model.EtsScene import org.jacodb.ets.utils.Maybe import org.jacodb.ets.utils.callExpr -import java.util.concurrent.atomic.AtomicInteger -import kotlin.math.roundToInt private val logger = KotlinLogging.logger {} @@ -91,136 +89,8 @@ class EtsApplicationGraphImpl( private val cacheClassWithIdealSignature: MutableMap> = hashMapOf() private val cacheMethodWithIdealSignature: MutableMap> = hashMapOf() - private val cachePartiallyMatchFailedCallees: MutableSet = hashSetOf() - private val cachePartiallyMultipleMatchesCallees: MutableMap> = hashMapOf() private val cachePartiallyMatchedCallees: MutableMap> = hashMapOf() - data class CalleeStats( - val stmtWithNoCall: AtomicInteger = AtomicInteger(0), - val unknownConstructor: AtomicInteger = AtomicInteger(0), - val constructorOfClassNotFound: AtomicInteger = AtomicInteger(0), - val resolvedConstructor: AtomicInteger = AtomicInteger(0), - val resolvedByTotallyMatchedCache: AtomicInteger = AtomicInteger(0), - val cachedTotalMatchAsUnknown: AtomicInteger = AtomicInteger(0), - val lookupWithIdealSignatureFailed: AtomicInteger = AtomicInteger(0), - val resolvedByIdealSignature: AtomicInteger = AtomicInteger(0), - val resolvedByNeighbour: AtomicInteger = AtomicInteger(0), - val resolvedByPartiallyMatchedCache: AtomicInteger = AtomicInteger(0), - val cachedPartialMatchAsUnknown: AtomicInteger = AtomicInteger(0), - val noPartialMatch: AtomicInteger = AtomicInteger(0), - val multiplePartialMatch: AtomicInteger = AtomicInteger(0), - val partialMatchFound: AtomicInteger = AtomicInteger(0), - ) { - val total: Int - get() = listOf( - unknownConstructor, - constructorOfClassNotFound, - resolvedConstructor, - resolvedByTotallyMatchedCache, - cachedTotalMatchAsUnknown, - lookupWithIdealSignatureFailed, - resolvedByIdealSignature, - resolvedByNeighbour, - resolvedByPartiallyMatchedCache, - cachedPartialMatchAsUnknown, - noPartialMatch, - multiplePartialMatch, - partialMatchFound - ).sumOf { it.get() } - - val resolved: Int - get() = listOf( - resolvedConstructor, - resolvedByNeighbour, - resolvedByIdealSignature, - resolvedByPartiallyMatchedCache, - resolvedByTotallyMatchedCache, - partialMatchFound - ).sumOf { it.get() } - - val unresolved: Int - get() = total - resolved - - val constructors: Int - get() = listOf( - unknownConstructor, - resolvedConstructor, - constructorOfClassNotFound - ).sumOf { it.get() } - - val resolvedByCache: Int - get() = listOf( - resolvedByTotallyMatchedCache, - resolvedByPartiallyMatchedCache - ).sumOf { it.get() } - - val unresolvedByCache: Int - get() = listOf( - cachedTotalMatchAsUnknown, - cachedPartialMatchAsUnknown - ).sumOf { it.get() } - - val cacheHits: Int - get() = resolvedByCache + unresolvedByCache - - val cacheMisses: Int - get() = total - cacheHits - - private fun show(description: String, property: AtomicInteger): String { - if (total == 0) { - return "[N/A%] $description ${property.get()}" - } - val percent = 100.0 * property.get() / total - return "[${percent.roundToInt()}%] $description: ${property.get()}" - } - - private fun show(description: String, property: Int): String { - if (total == 0) { - return "[N/A%] $description $property" - } - val percent = 100.0 * property / total - return "[${percent.roundToInt()}%] $description: $property" - } - - override fun toString(): String { - return """ - CALLEE RESOLVER STATS: - - Errors: - not a call expression: ${stmtWithNoCall.incrementAndGet()} - - Constructors: - ${show("unknown class signature", unknownConstructor)} - ${show("classpath lookup failed", constructorOfClassNotFound)} - ${show("successfully resolved", resolvedConstructor)} - - Perfect matches: - ${show("resolved by cache", resolvedByTotallyMatchedCache)} - ${show("cached as unknown", cachedTotalMatchAsUnknown)} - ${show("lookup failed", lookupWithIdealSignatureFailed)} - ${show("successfully resolved", resolvedByIdealSignature)} - - Neighbours: - ${show("resolved by neighbour", resolvedByNeighbour)} - - Partial matches: - ${show("resolved by cache", resolvedByPartiallyMatchedCache)} - ${show("cached as unknown", cachedPartialMatchAsUnknown)} - ${show("no partial match", noPartialMatch)} - ${show("multiple partial match", multiplePartialMatch)} - ${show("successfully resolved", partialMatchFound)} - ------------------------------------------------- - Summary: - + ${show("Total", total)} - - + ${show("Resolved", resolved)} - + ${show("Unresolved", unresolved)} - - + ${show("Constructors", constructors)} - - + ${show("Cache hits", cacheHits)} - + ${show("Cache misses", cacheMisses)} - """.trimIndent() - } - } - - val stats = CalleeStats() - private fun lookupClassWithIdealSignature(signature: EtsClassSignature): Maybe { require(signature.isIdeal()) @@ -244,10 +114,8 @@ class EtsApplicationGraphImpl( } override fun callees(node: EtsStmt): Sequence { - val expr = node.callExpr ?: run { - stats.stmtWithNoCall.incrementAndGet() - return emptySequence() - } + val expr = node.callExpr ?: return emptySequence() + val callee = expr.method // Note: the resolving code below expects that at least the current method signature is known. @@ -259,7 +127,6 @@ class EtsApplicationGraphImpl( if (callee.name == CONSTRUCTOR_NAME) { if (!callee.enclosingClass.isIdeal()) { // Constructor signature is garbage. Sorry, can't do anything in such case. - stats.unknownConstructor.incrementAndGet() return emptySequence() } @@ -268,10 +135,8 @@ class EtsApplicationGraphImpl( val cls = lookupClassWithIdealSignature(callee.enclosingClass) if (cls.isSome) { - stats.resolvedConstructor.incrementAndGet() return sequenceOf(cls.getOrThrow().ctor) } else { - stats.constructorOfClassNotFound.incrementAndGet() return emptySequence() } } @@ -281,10 +146,8 @@ class EtsApplicationGraphImpl( if (callee in cacheMethodWithIdealSignature) { val resolved = cacheMethodWithIdealSignature.getValue(callee) if (resolved.isSome) { - stats.resolvedByTotallyMatchedCache.incrementAndGet() return sequenceOf(resolved.getOrThrow()) } else { - stats.cachedTotalMatchAsUnknown.incrementAndGet() return emptySequence() } } @@ -300,13 +163,11 @@ class EtsApplicationGraphImpl( } if (resolved.none()) { cacheMethodWithIdealSignature[callee] = Maybe.none() - stats.lookupWithIdealSignatureFailed.incrementAndGet() return emptySequence() } val r = resolved.singleOrNull() ?: error("Multiple methods with the same complete signature: ${resolved.toList()}") cacheMethodWithIdealSignature[callee] = Maybe.some(r) - stats.resolvedByIdealSignature.incrementAndGet() return sequenceOf(r) } @@ -331,19 +192,12 @@ class EtsApplicationGraphImpl( val s = neighbors.singleOrNull() ?: error("Multiple methods with the same name: $neighbors") cachePartiallyMatchedCallees[callee] = listOf(s) - stats.resolvedByNeighbour.incrementAndGet() return sequenceOf(s) } // NOTE: cache lookup MUST be performed AFTER trying to match the neighbour! if (callee in cachePartiallyMatchedCallees) { - val s = cachePartiallyMatchedCallees.getValue(callee).asSequence() - if (s.none()) { - stats.cachedPartialMatchAsUnknown.incrementAndGet() - } else { - stats.resolvedByPartiallyMatchedCache.incrementAndGet() - } - return s + return cachePartiallyMatchedCallees.getValue(callee).asSequence() } // If the neighbour match failed, @@ -359,17 +213,14 @@ class EtsApplicationGraphImpl( .toList() if (resolved.isEmpty()) { cachePartiallyMatchedCallees[callee] = emptyList() - stats.noPartialMatch.incrementAndGet() return emptySequence() } val r = resolved.singleOrNull() ?: run { logger.warn { "Multiple methods with the same partial signature '${callee}': $resolved" } cachePartiallyMatchedCallees[callee] = emptyList() - stats.multiplePartialMatch.incrementAndGet() return emptySequence() } cachePartiallyMatchedCallees[callee] = listOf(r) - stats.partialMatchFound.incrementAndGet() return sequenceOf(r) } From f7ee59bf30461e6a8453af32950914d31b5cf922 Mon Sep 17 00:00:00 2001 From: Alexey Menshutin Date: Wed, 25 Dec 2024 12:54:08 +0300 Subject: [PATCH 120/120] Fix failed complication --- .../test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt index 7eae241b5..49c1d8c9e 100644 --- a/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt +++ b/jacodb-ets/src/test/kotlin/org/jacodb/ets/test/EtsFromJsonTest.kt @@ -124,8 +124,8 @@ class EtsFromJsonTest { ) val prefix = "$PROJECT_PATH/etsir" val project = loadEtsProjectFromResources(modules, prefix) - println("Classes: ${project.classes.size}") - for (cls in project.classes) { + println("Classes: ${project.projectClasses.size}") + for (cls in project.projectClasses) { println("= ${cls.signature} with ${cls.methods.size} methods:") for (method in cls.methods) { println(" - ${method.signature}") @@ -178,8 +178,8 @@ class EtsFromJsonTest { val project = loadEtsProjectFromResources(modules, "/projects/$projectName/etsir") logger.info { buildString { - appendLine("Loaded project with ${project.classes.size} classes and ${project.classes.sumOf { it.methods.size }} methods") - for (cls in project.classes) { + appendLine("Loaded project with ${project.projectClasses.size} classes and ${project.projectClasses.sumOf { it.methods.size }} methods") + for (cls in project.projectClasses) { appendLine("= ${cls.signature} with ${cls.methods.size} methods:") for (method in cls.methods) { appendLine(" - ${method.signature}")