From c40bba4a907b5143328c334ca0bacdd478ae56a2 Mon Sep 17 00:00:00 2001 From: Zofia Wiora Date: Fri, 22 Aug 2025 16:09:31 +0200 Subject: [PATCH] KTL-2962 Compilation with BTA for JVM --- build.gradle.kts | 6 +- common/build.gradle.kts | 4 +- gradle/libs.versions.toml | 6 +- .../compiler/components/CompilationLogger.kt | 87 ++++++++++++++++ .../compiler/components/KotlinCompiler.kt | 99 +++++++++++++++---- .../server/configuration/BuildToolsConfig.kt | 19 ++++ .../com/compiler/server/model/TextInterval.kt | 2 +- .../server/service/KotlinProjectExecutor.kt | 3 - .../server/CompilerArgumentsEndpointTest.kt | 1 - .../compose-wasm-expected-compiler-args.json | 4 +- .../js-expected-compiler-args.json | 4 +- .../jvm-expected-compiler-args.json | 4 +- .../wasm-expected-compiler-args.json | 4 +- 13 files changed, 203 insertions(+), 40 deletions(-) create mode 100644 src/main/kotlin/com/compiler/server/compiler/components/CompilationLogger.kt create mode 100644 src/main/kotlin/com/compiler/server/configuration/BuildToolsConfig.kt diff --git a/build.gradle.kts b/build.gradle.kts index 7dea54376..8c54d0408 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -44,13 +44,15 @@ dependencies { implementation(libs.gson) implementation(libs.kotlinx.serialization.json) implementation(libs.kotlin.compiler.arguments.description) - implementation(libs.kotlin.tooling.core) implementation(libs.junit) implementation(libs.logback.logstash.encoder) implementation(libs.kotlin.reflect) implementation(libs.kotlin.stdlib) - implementation(libs.kotlin.compiler) implementation(libs.kotlin.script.runtime) + implementation(libs.kotlin.build.tools.api) + implementation(libs.kotlin.build.tools.impl) + implementation(libs.kotlin.compiler.embeddable) + implementation(libs.kotlin.tooling.core) implementation(project(":executors", configuration = "default")) implementation(project(":common", configuration = "default")) implementation(project(":dependencies")) diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 8ee7fd2b7..7a5f934a9 100644 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -3,5 +3,5 @@ plugins { } dependencies { - implementation(libs.kotlin.compiler) -} + implementation(libs.kotlin.compiler.embeddable) +} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 39c69e386..b1e5ca5d7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,5 @@ [versions] -kotlin = "2.3.0-dev-9317" +kotlin = "2.3.0-dev-9673" spring-boot = "3.5.6" spring-dependency-managment = "1.1.7" springdoc = "2.8.13" @@ -25,7 +25,9 @@ kotlin-stdlib-js = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib-js", kotlin-stdlib-wasm-js = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib-wasm-js", version.ref = "kotlin" } kotlin-test = { group = "org.jetbrains.kotlin", name = "kotlin-test", version.ref = "kotlin" } kotlin-test-junit = { group = "org.jetbrains.kotlin", name = "kotlin-test-junit", version.ref = "kotlin" } -kotlin-compiler = { group = "org.jetbrains.kotlin", name = "kotlin-compiler", version.ref = "kotlin" } +kotlin-build-tools-api = { group = "org.jetbrains.kotlin", name = "kotlin-build-tools-api", version.ref = "kotlin"} +kotlin-build-tools-impl = { group = "org.jetbrains.kotlin", name = "kotlin-build-tools-impl", version.ref = "kotlin"} +kotlin-compiler-embeddable = { group = "org.jetbrains.kotlin", name = "kotlin-compiler-embeddable", version.ref = "kotlin" } kotlin-tooling-core = { group = "org.jetbrains.kotlin", name = "kotlin-tooling-core", version.ref = "kotlin" } kotlin-compiler-arguments-description = { group = "org.jetbrains.kotlin", name = "kotlin-compiler-arguments-description", version.ref = "kotlin" } kotlin-script-runtime = { group = "org.jetbrains.kotlin", name = "kotlin-script-runtime", version.ref = "kotlin" } diff --git a/src/main/kotlin/com/compiler/server/compiler/components/CompilationLogger.kt b/src/main/kotlin/com/compiler/server/compiler/components/CompilationLogger.kt new file mode 100644 index 000000000..7fe739217 --- /dev/null +++ b/src/main/kotlin/com/compiler/server/compiler/components/CompilationLogger.kt @@ -0,0 +1,87 @@ +package com.compiler.server.compiler.components + +import com.compiler.server.model.ErrorDescriptor +import com.compiler.server.model.ProjectSeveriry +import com.compiler.server.model.TextInterval +import org.jetbrains.kotlin.buildtools.api.KotlinLogger + + +/** + * This custom implementation of Kotlin Logger is needed for sending compilation logs to the user + * on the frontend instead of printing them on the stderr. CompilationLogger extracts data from logs + * and saves it in [compilationLogs] map, so that compilation messages can be later displayed to + * the user, and their position can be marked in their code. + + * KotlinLogger interface will be changed in the future to contain more log details. + * Implementation of the CompilationLogger should be therefore updated after KT-80963 is implemented. + * + * @property isDebugEnabled A flag to indicate whether debug-level logging is enabled for the logger. + * If true, all messages are printed to the standard output. + */ +class CompilationLogger( + override val isDebugEnabled: Boolean = false, +) : KotlinLogger { + + /** + * Stores a collection of compilation logs organized by file paths. + * + * The map keys represent file paths as strings, and the associated values are mutable lists of + * `ErrorDescriptor` objects containing details about compilation issues, such as error messages, + * intervals, severity, and optional class names. + */ + var compilationLogs: Map> = emptyMap() + + override fun debug(msg: String) { + if (isDebugEnabled) println("[DEBUG] $msg") + } + + override fun error(msg: String, throwable: Throwable?) { + if (isDebugEnabled) println("[ERROR] $msg" + (throwable?.let { ": ${it.message}" } ?: "")) + try { + addCompilationLog(msg, ProjectSeveriry.ERROR, classNameOverride = null) + } catch (_: Exception) {} + } + + override fun info(msg: String) { + if (isDebugEnabled) println("[INFO] $msg") + } + + override fun lifecycle(msg: String) { + if (isDebugEnabled) println("[LIFECYCLE] $msg") + } + + override fun warn(msg: String, throwable: Throwable?) { + if (isDebugEnabled) println("[WARN] $msg" + (throwable?.let { ": ${it.message}" } ?: "")) + try { + addCompilationLog(msg, ProjectSeveriry.WARNING, classNameOverride = "WARNING") + } catch (_: Exception) {} + } + + + /** + * Adds a compilation log entry to the `compilationLogs` map based on the string log. + * + * @param msg The raw log message containing information about the compilation event, + * including the file path and error details. + * @param severity The severity level of the compilation event, represented by the `ProjectSeveriry` enum. + * @param classNameOverride An optional override for the class name that will be recorded in the log. + * If null, it will be derived from the file path in the message. + */ + private fun addCompilationLog(msg: String, severity: ProjectSeveriry, classNameOverride: String?) { + val path = msg.split(" ")[0] + val className = path.split("/").last().split(".").first() + val message = msg.split(path)[1].drop(1) + val splitPath = path.split(":") + val line = splitPath[splitPath.size - 4].toInt() - 1 + val ch = splitPath[splitPath.size - 3].toInt() - 1 + val endLine = splitPath[splitPath.size - 2].toInt() - 1 + val endCh = splitPath[splitPath.size - 1].toInt() - 1 + val ed = ErrorDescriptor( + TextInterval(TextInterval.TextPosition(line, ch), TextInterval.TextPosition(endLine, endCh)), + message, + severity, + classNameOverride ?: className + ) + compilationLogs["$className.kt"]?.add(ed) + } +} diff --git a/src/main/kotlin/com/compiler/server/compiler/components/KotlinCompiler.kt b/src/main/kotlin/com/compiler/server/compiler/components/KotlinCompiler.kt index 835282321..e97522af0 100644 --- a/src/main/kotlin/com/compiler/server/compiler/components/KotlinCompiler.kt +++ b/src/main/kotlin/com/compiler/server/compiler/components/KotlinCompiler.kt @@ -2,6 +2,7 @@ package com.compiler.server.compiler.components import com.compiler.server.executor.CommandLineArgument import com.compiler.server.executor.JavaExecutor +import com.compiler.server.model.CompilerDiagnostics import com.compiler.server.model.ExtendedCompilerArgument import com.compiler.server.model.JvmExecutionResult import com.compiler.server.model.OutputDirectory @@ -12,7 +13,9 @@ import com.compiler.server.utils.CompilerArgumentsUtil import component.KotlinEnvironment import executors.JUnitExecutors import executors.JavaRunnerExecutor -import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi +import org.jetbrains.kotlin.buildtools.api.KotlinToolchains +import org.jetbrains.kotlin.buildtools.api.jvm.JvmPlatformToolchain import org.jetbrains.org.objectweb.asm.ClassReader import org.jetbrains.org.objectweb.asm.ClassReader.* import org.jetbrains.org.objectweb.asm.ClassVisitor @@ -63,7 +66,12 @@ class KotlinCompiler( ?.joinToString("\n\n") } - fun run(files: List, addByteCode: Boolean, args: String, userCompilerArguments: Map): JvmExecutionResult { + fun run( + files: List, + addByteCode: Boolean, + args: String, + userCompilerArguments: Map + ): JvmExecutionResult { return execute(files, addByteCode, userCompilerArguments) { output, compiled -> val mainClass = JavaRunnerExecutor::class.java.name val compiledMainClass = when (compiled.mainClasses.size) { @@ -86,7 +94,11 @@ class KotlinCompiler( } } - fun test(files: List, addByteCode: Boolean, userCompilerArguments: Map): JvmExecutionResult { + fun test( + files: List, + addByteCode: Boolean, + userCompilerArguments: Map + ): JvmExecutionResult { return execute(files, addByteCode, userCompilerArguments) { output, _ -> val mainClass = JUnitExecutors::class.java.name javaExecutor.execute(argsFrom(mainClass, output, listOf(output.path.toString()))) @@ -94,29 +106,74 @@ class KotlinCompiler( } } - @OptIn(ExperimentalPathApi::class) - fun compile(files: List, userCompilerArguments: Map): CompilationResult = usingTempDirectory { inputDir -> - val ioFiles = files.writeToIoFiles(inputDir) - usingTempDirectory { outputDir -> - val arguments = ioFiles.map { it.absolutePathString() } + - compilerArgumentsUtil.convertCompilerArgumentsToCompilationString(jvmCompilerArguments, compilerArgumentsUtil.PREDEFINED_JVM_ARGUMENTS, userCompilerArguments) + - listOf("-d", outputDir.absolutePathString()) - K2JVMCompiler().tryCompilation(inputDir, ioFiles, arguments) { - val outputFiles = buildMap { - outputDir.visitFileTree { - onVisitFile { file, _ -> - put(file.relativeTo(outputDir).pathString, file.readBytes()) - FileVisitResult.CONTINUE - } + fun compile(files: List, userCompilerArguments: Map): CompilationResult = + usingTempDirectory { inputDir -> + val ioFiles = files.writeToIoFiles(inputDir) + usingTempDirectory { outputDir -> + val arguments = ioFiles.map { it.absolutePathString() } + + compilerArgumentsUtil.convertCompilerArgumentsToCompilationString( + jvmCompilerArguments, + compilerArgumentsUtil.PREDEFINED_JVM_ARGUMENTS, + userCompilerArguments + ) + val result = compileWithToolchain(inputDir, outputDir, arguments) + return@usingTempDirectory result + } + } + + @OptIn(ExperimentalPathApi::class, ExperimentalBuildToolsApi::class, ExperimentalBuildToolsApi::class) + private fun compileWithToolchain( + inputDir: Path, + outputDir: Path, + arguments: List + ): CompilationResult { + val sources = inputDir.listDirectoryEntries() + + val logger = CompilationLogger() + logger.compilationLogs = sources + .filter { it.name.endsWith(".kt") } + .associate { it.name to mutableListOf() } + + val toolchains = KotlinToolchains.loadImplementation(ClassLoader.getSystemClassLoader()) + val jvmToolchain = toolchains.getToolchain(JvmPlatformToolchain::class.java) + val operation = jvmToolchain.createJvmCompilationOperation(sources, outputDir) + operation.compilerArguments.applyArgumentStrings(arguments) + + toolchains.createBuildSession().use { session -> + val result = try { + session.executeOperation(operation, toolchains.createInProcessExecutionPolicy(), logger) + } catch (e: Exception) { + throw Exception("Exception executing compilation operation", e) + } + return toCompilationResult(result, logger, outputDir) + } + } + + private fun toCompilationResult( + buildResult: org.jetbrains.kotlin.buildtools.api.CompilationResult, + logger: CompilationLogger, + outputDir: Path, + ): CompilationResult = when (buildResult) { + org.jetbrains.kotlin.buildtools.api.CompilationResult.COMPILATION_SUCCESS -> { + val compilerDiagnostics = CompilerDiagnostics(logger.compilationLogs) + val outputFiles = buildMap { + outputDir.visitFileTree { + onVisitFile { file, _ -> + put(file.relativeTo(outputDir).pathString, file.readBytes()) + FileVisitResult.CONTINUE } } - val mainClasses = findMainClasses(outputFiles) - JvmClasses( + } + Compiled( + compilerDiagnostics = compilerDiagnostics, + result = JvmClasses( files = outputFiles, - mainClasses = mainClasses, + mainClasses = findMainClasses(outputFiles), ) - } + ) } + + else -> NotCompiled(CompilerDiagnostics(logger.compilationLogs)) } private fun findMainClasses(outputFiles: Map): Set = diff --git a/src/main/kotlin/com/compiler/server/configuration/BuildToolsConfig.kt b/src/main/kotlin/com/compiler/server/configuration/BuildToolsConfig.kt new file mode 100644 index 000000000..cdd8f883e --- /dev/null +++ b/src/main/kotlin/com/compiler/server/configuration/BuildToolsConfig.kt @@ -0,0 +1,19 @@ +package com.compiler.server.configuration + +import org.springframework.context.annotation.Configuration + +@Configuration +class BuildToolsConfig { + init { + /** + * This flag is used by KotlinMessageRenderer in kotlin-build-tools-api to properly format + * returned log messages during compilation. When this flag is set, the whole position of + * a warning/error is returned instead of only the beginning. We need this behavior to + * process messages in KotlinLogger and then correctly mark errors on the frontend. + * + * Setting this property should be removed after KT-80963 is implemented, as KotlinLogger + * will return the full position of an error by default then. + */ + System.setProperty("org.jetbrains.kotlin.buildtools.logger.extendedLocation", "true") + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/compiler/server/model/TextInterval.kt b/src/main/kotlin/com/compiler/server/model/TextInterval.kt index ecaff34f1..69e120835 100644 --- a/src/main/kotlin/com/compiler/server/model/TextInterval.kt +++ b/src/main/kotlin/com/compiler/server/model/TextInterval.kt @@ -1,6 +1,6 @@ package com.compiler.server.model -import com.intellij.openapi.editor.Document +import org.jetbrains.kotlin.com.intellij.openapi.editor.Document data class TextInterval(val start: TextPosition, val end: TextPosition) { data class TextPosition(val line: Int, val ch: Int) : Comparable { diff --git a/src/main/kotlin/com/compiler/server/service/KotlinProjectExecutor.kt b/src/main/kotlin/com/compiler/server/service/KotlinProjectExecutor.kt index aba17d5c5..ebb6e3e73 100644 --- a/src/main/kotlin/com/compiler/server/service/KotlinProjectExecutor.kt +++ b/src/main/kotlin/com/compiler/server/service/KotlinProjectExecutor.kt @@ -7,7 +7,6 @@ import com.compiler.server.model.JsCompilerArguments import com.compiler.server.model.bean.VersionInfo import component.KotlinEnvironment import model.Completion -import org.junit.Ignore import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -45,7 +44,6 @@ class KotlinProjectExecutor( } // TODO(Dmitrii Krasnov): implement this method in KTL-2807 - @Ignore fun complete(project: Project, line: Int, character: Int): List { return emptyList() } @@ -117,5 +115,4 @@ class KotlinProjectExecutor( getVersion().version ) } - } diff --git a/src/test/kotlin/com/compiler/server/CompilerArgumentsEndpointTest.kt b/src/test/kotlin/com/compiler/server/CompilerArgumentsEndpointTest.kt index bdfa0bd93..f2f6dcb61 100644 --- a/src/test/kotlin/com/compiler/server/CompilerArgumentsEndpointTest.kt +++ b/src/test/kotlin/com/compiler/server/CompilerArgumentsEndpointTest.kt @@ -4,7 +4,6 @@ import com.compiler.server.model.ProjectType import com.compiler.server.model.bean.VersionInfo import com.fasterxml.jackson.databind.ObjectMapper import component.KotlinEnvironment -import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource import org.springframework.beans.factory.annotation.Autowired diff --git a/src/test/resources/compiler-arguments/compose-wasm-expected-compiler-args.json b/src/test/resources/compiler-arguments/compose-wasm-expected-compiler-args.json index af04c2b44..74f34139c 100644 --- a/src/test/resources/compiler-arguments/compose-wasm-expected-compiler-args.json +++ b/src/test/resources/compiler-arguments/compose-wasm-expected-compiler-args.json @@ -1388,8 +1388,8 @@ "description": "Use an updated version of the exception proposal with try_table.", "type": { "type": "com.compiler.server.model.BooleanExtendedCompilerArgumentValue", - "isNullable": false, - "defaultValue": false + "isNullable": true, + "defaultValue": null }, "disabled": false, "predefinedValues": null diff --git a/src/test/resources/compiler-arguments/js-expected-compiler-args.json b/src/test/resources/compiler-arguments/js-expected-compiler-args.json index e07972f9a..2dc940eb9 100644 --- a/src/test/resources/compiler-arguments/js-expected-compiler-args.json +++ b/src/test/resources/compiler-arguments/js-expected-compiler-args.json @@ -1386,8 +1386,8 @@ "description": "Use an updated version of the exception proposal with try_table.", "type": { "type": "com.compiler.server.model.BooleanExtendedCompilerArgumentValue", - "isNullable": false, - "defaultValue": false + "isNullable": true, + "defaultValue": null }, "disabled": false, "predefinedValues": null diff --git a/src/test/resources/compiler-arguments/jvm-expected-compiler-args.json b/src/test/resources/compiler-arguments/jvm-expected-compiler-args.json index 86620d64f..aff5fba50 100644 --- a/src/test/resources/compiler-arguments/jvm-expected-compiler-args.json +++ b/src/test/resources/compiler-arguments/jvm-expected-compiler-args.json @@ -1299,7 +1299,7 @@ { "name": "jvm-target", "shortName": null, - "description": "The target version of the generated JVM bytecode (1.8 and 9–24), with 1.8 as the default.", + "description": "The target version of the generated JVM bytecode (1.8 and 9–25), with 1.8 as the default.", "type": { "type": "com.compiler.server.model.StringExtendedCompilerArgumentValue", "isNullable": true, @@ -1779,7 +1779,7 @@ { "name": "Xjdk-release", "shortName": null, - "description": "Compile against the specified JDK API version, similarly to javac's '-release'. This requires JDK 9 or newer.\nThe supported versions depend on the JDK used; for JDK 17+, the supported versions are 1.8 and 9–24.\nThis also sets the value of '-jvm-target' to be equal to the selected JDK version.", + "description": "Compile against the specified JDK API version, similarly to javac's '-release'. This requires JDK 9 or newer.\nThe supported versions depend on the JDK used; for JDK 17+, the supported versions are 1.8 and 9–25.\nThis also sets the value of '-jvm-target' to be equal to the selected JDK version.", "type": { "type": "com.compiler.server.model.StringExtendedCompilerArgumentValue", "isNullable": true, diff --git a/src/test/resources/compiler-arguments/wasm-expected-compiler-args.json b/src/test/resources/compiler-arguments/wasm-expected-compiler-args.json index 66957a971..31a73fc07 100644 --- a/src/test/resources/compiler-arguments/wasm-expected-compiler-args.json +++ b/src/test/resources/compiler-arguments/wasm-expected-compiler-args.json @@ -1386,8 +1386,8 @@ "description": "Use an updated version of the exception proposal with try_table.", "type": { "type": "com.compiler.server.model.BooleanExtendedCompilerArgumentValue", - "isNullable": false, - "defaultValue": false + "isNullable": true, + "defaultValue": null }, "disabled": false, "predefinedValues": null