From 617a66c1e1b7b92f9cd582ccbf956d56e6c19811 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Fri, 27 Jun 2025 16:05:50 +0200 Subject: [PATCH 1/9] bumping to Kotlin 2.2 as well as some other dependencies --- dataframe-arrow/api/dataframe-arrow.api | 17 +++++++++-------- gradle.properties | 4 ++++ gradle/libs.versions.toml | 14 +++++++------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/dataframe-arrow/api/dataframe-arrow.api b/dataframe-arrow/api/dataframe-arrow.api index f9a850a022..2d2e8f1c0f 100644 --- a/dataframe-arrow/api/dataframe-arrow.api +++ b/dataframe-arrow/api/dataframe-arrow.api @@ -53,14 +53,15 @@ public abstract interface class org/jetbrains/kotlinx/dataframe/io/ArrowWriter : public abstract fun getMismatchSubscriber ()Lkotlin/jvm/functions/Function1; public abstract fun getMode ()Lorg/jetbrains/kotlinx/dataframe/io/ArrowWriter$Mode; public abstract fun getTargetSchema ()Lorg/apache/arrow/vector/types/pojo/Schema; - public abstract fun saveArrowFeatherToByteArray ()[B - public abstract fun saveArrowIPCToByteArray ()[B - public abstract fun writeArrowFeather (Ljava/io/File;)V - public abstract fun writeArrowFeather (Ljava/io/OutputStream;)V - public abstract fun writeArrowFeather (Ljava/nio/channels/WritableByteChannel;)V - public abstract fun writeArrowIPC (Ljava/io/File;Z)V - public abstract fun writeArrowIPC (Ljava/io/OutputStream;)V - public abstract fun writeArrowIPC (Ljava/nio/channels/WritableByteChannel;)V + public fun saveArrowFeatherToByteArray ()[B + public fun saveArrowIPCToByteArray ()[B + public fun writeArrowFeather (Ljava/io/File;)V + public fun writeArrowFeather (Ljava/io/OutputStream;)V + public fun writeArrowFeather (Ljava/nio/channels/WritableByteChannel;)V + public fun writeArrowIPC (Ljava/io/File;Z)V + public fun writeArrowIPC (Ljava/io/OutputStream;)V + public fun writeArrowIPC (Ljava/nio/channels/WritableByteChannel;)V + public static synthetic fun writeArrowIPC$default (Lorg/jetbrains/kotlinx/dataframe/io/ArrowWriter;Ljava/io/File;ZILjava/lang/Object;)V } public final class org/jetbrains/kotlinx/dataframe/io/ArrowWriter$Companion { diff --git a/gradle.properties b/gradle.properties index 8df75a945a..191eb8fd32 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,7 @@ version=1.0.0 jupyterApiTCRepo= kotlin.jupyter.add.scanner=false org.gradle.jvmargs=-Xmx4G -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 + # build.number.detection=false # build.number=0.8.0 @@ -12,6 +13,9 @@ org.gradle.jvmargs=-Xmx4G -Duser.language=en -Duser.country=US -Dfile.encoding=U # KSP plugin in the modules that use it. kotlin.dataframe.add.ksp=false +# We don't support ksp2 +ksp.useKSP2=false + # Enables debug mode for dataframe. # This can make certain tests and checks run that should not be run in production. # It can also be turned on from the command line with `-Pkotlin.dataframe.debug=true` diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d53a851a46..8a723c9303 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,15 +1,15 @@ [versions] -ksp = "2.0.20-1.0.24" +ksp = "2.2.0-2.0.2" kotlinJupyter = "0.12.0-383" ktlint = "12.3.0" # make sure to sync both manually with :generator module -kotlin = "2.0.20" # needs jupyter compatibility with Kotlin 2.1 to update -kotlinpoet = "1.18.1" +kotlin = "2.2.0" # needs jupyter compatibility with Kotlin 2.1 to update +kotlinpoet = "2.2.0" -dokka = "1.9.20" -libsPublisher = "1.9.23-dev-45" +dokka = "2.0.0" +libsPublisher = "2.2.0-rc3-dev-57" # "Bootstrap" version of the dataframe, used in the build itself to generate @DataSchema APIs, # dogfood Gradle / KSP plugins in tests and idea-examples modules @@ -48,14 +48,14 @@ jsoup = "1.18.3" arrow = "18.1.0" kodex = "0.4.4" simpleGit = "2.2.1" -dependencyVersions = "0.51.0" +dependencyVersions = "0.52.0" plugin-publish = "1.3.0" shadow = "8.3.5" android-gradle-api = "7.3.1" # need to revise our tests to update ktor = "3.0.1" # needs jupyter compatibility with Kotlin 2.1 to update kotlin-compile-testing = "1.6.0" duckdb = "1.1.3" -buildconfig = "5.5.1" +buildconfig = "5.6.7" benchmark = "0.4.12" geotools = "32.1" From 9b4f5dcd912d1b0d26c2d6bc61612c37c028af2e Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Fri, 27 Jun 2025 16:08:46 +0200 Subject: [PATCH 2/9] tiny API fixes for kotlin 2.2 in core --- .../main/kotlin/org/jetbrains/kotlinx/dataframe/api/groupBy.kt | 3 ++- .../main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt | 3 ++- .../src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/groupBy.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/groupBy.kt index cea5bdc489..5c670a77ef 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/groupBy.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/groupBy.kt @@ -115,4 +115,5 @@ public class ReducedGroupBy( } @PublishedApi -internal fun GroupBy.reduce(reducer: Selector, DataRow?>) = ReducedGroupBy(this, reducer) +internal fun GroupBy.reduce(reducer: Selector, DataRow?>): ReducedGroupBy = + ReducedGroupBy(this, reducer) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt index acfc2145ea..286a40e5b7 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt @@ -268,7 +268,8 @@ public class ReducedPivot( } @PublishedApi -internal fun Pivot.reduce(reducer: Selector, DataRow?>) = ReducedPivot(this, reducer) +internal fun Pivot.reduce(reducer: Selector, DataRow?>): ReducedPivot = + ReducedPivot(this, reducer) @PublishedApi internal inline fun Pivot.delegate(crossinline body: PivotGroupBy.() -> DataFrame): DataRow = diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt index 87176ca7e4..6994580bab 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/html.kt @@ -533,7 +533,7 @@ public fun DataFrame.toHTML( configuration: DisplayConfiguration = DisplayConfiguration.DEFAULT, cellRenderer: CellRenderer = DefaultCellRenderer, getFooter: (DataFrame) -> String? = { "DataFrame [${it.size}]" }, -) = toHtml(configuration, cellRenderer, getFooter) +): DataFrameHtmlData = toHtml(configuration, cellRenderer, getFooter) @Deprecated(TO_STANDALONE_HTML, ReplaceWith(TO_STANDALONE_HTML_REPLACE), DeprecationLevel.ERROR) public fun DataFrame.toStandaloneHTML( From 5893965a94dd8b5ca88933389c53ce914ca1627a Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Fri, 27 Jun 2025 16:09:35 +0200 Subject: [PATCH 3/9] tiny fixes for kotlin 2.2 in gradle plugin --- build.gradle.kts | 2 + .../dataframe-gradle-plugin/build.gradle.kts | 39 +++++++++++++++---- .../dataframe/gradle/SchemaGeneratorPlugin.kt | 15 +++++-- .../dataframe/gradle/GradleRunnerDsl.kt | 2 + .../jetbrains/dataframe/gradle/TestData.kt | 4 +- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index aba40c6a4a..c156c81955 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -155,6 +155,7 @@ val modulesUsingJava11 = with(projects) { dataframeGeo, examples.ideaExamples.titanic, tests, + plugins.dataframeGradlePlugin, ) }.map { it.path } @@ -204,6 +205,7 @@ allprojects { configure { packageName = "org.jetbrains.kotlinx.dataframe" className = "BuildConfig" + buildConfigField("KOTLIN_VERSION", libs.versions.kotlin.asProvider().get()) buildConfigField("VERSION", "${project.version}") buildConfigField("DEBUG", findProperty("kotlin.dataframe.debug")?.toString()?.toBoolean() ?: false) } diff --git a/plugins/dataframe-gradle-plugin/build.gradle.kts b/plugins/dataframe-gradle-plugin/build.gradle.kts index a1af8b4021..f29b0b844b 100644 --- a/plugins/dataframe-gradle-plugin/build.gradle.kts +++ b/plugins/dataframe-gradle-plugin/build.gradle.kts @@ -1,9 +1,11 @@ plugins { `kotlin-dsl` - `java-gradle-plugin` `maven-publish` - alias(libs.plugins.plugin.publish) - alias(libs.plugins.ktlint) + with(libs.plugins) { + alias(buildconfig) + alias(plugin.publish) + alias(ktlint) + } } repositories { @@ -14,26 +16,34 @@ repositories { group = "org.jetbrains.kotlinx.dataframe" +buildscript { + dependencies { + classpath(embeddedKotlin("gradle-plugin")) + } +} + dependencies { api(libs.kotlin.reflect) implementation(projects.dataframe) // experimental implementation(projects.dataframeOpenapiGenerator) + compileOnly(embeddedKotlin("gradle-plugin")) implementation(libs.kotlin.gradle.plugin.api) - implementation(libs.kotlin.gradle.plugin) implementation(libs.serialization.core) implementation(libs.serialization.json) implementation(libs.ksp.gradle) implementation(libs.ksp.api) - testImplementation(libs.junit) + testImplementation(gradleTestKit()) + testImplementation(embeddedKotlin("test")) + testImplementation(embeddedKotlin("test-junit")) testImplementation(libs.kotestAssertions) testImplementation(libs.android.gradle.api) testImplementation(libs.android.gradle) + testImplementation(embeddedKotlin("gradle-plugin")) testImplementation(libs.ktor.server.netty) testImplementation(libs.h2db) - testImplementation(gradleApi()) } tasks.withType { @@ -100,7 +110,11 @@ val integrationTestConfiguration by configurations.creating { extendsFrom(configurations.testImplementation.get()) } -val integrationTestTask = task("integrationTest") { +tasks.pluginUnderTestMetadata { + pluginClasspath.from(integrationTestConfiguration) +} + +val integrationTestTask = tasks.register("integrationTest") { dependsOn(":plugins:symbol-processor:publishToMavenLocal") dependsOn(":dataframe-arrow:publishToMavenLocal") dependsOn(":dataframe-excel:publishToMavenLocal") @@ -126,3 +140,14 @@ val integrationTestTask = task("integrationTest") { } tasks.check { dependsOn(integrationTestTask) } + +// fixing linter + buildConfig +kotlin.sourceSets.create("buildConfigSources") { + kotlin.srcDir("build/generated/sources/buildConfig/main") +} +tasks.generateBuildConfig { + finalizedBy("runKtlintFormatOverBuildConfigSourcesSourceSet") +} +tasks.named("runKtlintCheckOverBuildConfigSourcesSourceSet") { + dependsOn(tasks.generateBuildConfig, "runKtlintFormatOverBuildConfigSourcesSourceSet") +} diff --git a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt index 7277f0f301..0ca158c041 100644 --- a/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt +++ b/plugins/dataframe-gradle-plugin/src/main/kotlin/org/jetbrains/dataframe/gradle/SchemaGeneratorPlugin.kt @@ -1,5 +1,7 @@ package org.jetbrains.dataframe.gradle +import com.google.devtools.ksp.gradle.KspAATask +import com.google.devtools.ksp.gradle.KspTask import com.google.devtools.ksp.gradle.KspTaskJvm import org.gradle.api.Plugin import org.gradle.api.Project @@ -8,13 +10,12 @@ import org.gradle.api.logging.LogLevel import org.gradle.api.tasks.TaskProvider import org.gradle.internal.logging.services.DefaultLoggingManager import org.gradle.kotlin.dsl.create -import org.gradle.kotlin.dsl.withType import org.jetbrains.kotlin.gradle.dsl.ExplicitApiMode import org.jetbrains.kotlin.gradle.dsl.KotlinAndroidProjectExtension import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.dsl.KotlinProjectExtension -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jetbrains.kotlin.gradle.tasks.BaseKotlinCompile import java.io.File import java.net.URL import java.nio.file.Path @@ -43,10 +44,16 @@ class SchemaGeneratorPlugin : Plugin { group = GROUP dependsOn(*generationTasks.toTypedArray()) } - tasks.withType(KspTaskJvm::class.java).configureEach { + tasks.withType(KspTask::class.java).configureEach { dependsOn(generateAll) + dependsOn(*generationTasks.toTypedArray()) + } + tasks.withType(KspAATask::class.java).configureEach { + error( + "Detected KSP2. This is not supported by the DataFrame Gradle/Ksp plugin. Add 'ksp.useKSP2=false' to 'gradle.properties'.", + ) } - tasks.withType { + tasks.withType(BaseKotlinCompile::class.java).configureEach { dependsOn(generateAll) } } diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/GradleRunnerDsl.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/GradleRunnerDsl.kt index a11ab964f5..fbf0134066 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/GradleRunnerDsl.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/GradleRunnerDsl.kt @@ -16,6 +16,8 @@ fun runGradleBuild( buildFile.writeText(build(buildDir)) val settingsFile = File(buildDir, "settings.gradle.kts") settingsFile.writeText(settingsGradle(buildDir)) + val propertiesFile = File(buildDir, "gradle.properties") + propertiesFile.writeText("ksp.useKSP2=false") return Build(buildDir, gradleRunner(buildDir, task).build()) } diff --git a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt index 12d36fe2cf..b03539e8c5 100644 --- a/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt +++ b/plugins/dataframe-gradle-plugin/src/test/kotlin/org/jetbrains/dataframe/gradle/TestData.kt @@ -1,5 +1,7 @@ package org.jetbrains.dataframe.gradle +import org.jetbrains.kotlinx.dataframe.BuildConfig + object TestData { val csvSample = @@ -15,5 +17,5 @@ object TestData { val jsonName = "test.json" - val kotlinVersion = "1.6.0" + val kotlinVersion = BuildConfig.KOTLIN_VERSION } From d14d8d3abeb81f141aad37c6d11443952acaa347 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Fri, 27 Jun 2025 16:11:11 +0200 Subject: [PATCH 4/9] bumping keywords-generator for kotlin 2.2, now using dynamic kotlin compiler version --- plugins/keywords-generator/build.gradle.kts | 10 ++++++---- plugins/keywords-generator/gradle.properties | 3 +-- .../dataframe/keywords/KeywordsGeneratorAction.kt | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/plugins/keywords-generator/build.gradle.kts b/plugins/keywords-generator/build.gradle.kts index fc192a8847..5f41cca787 100644 --- a/plugins/keywords-generator/build.gradle.kts +++ b/plugins/keywords-generator/build.gradle.kts @@ -1,5 +1,8 @@ +@file:OptIn(ExperimentalBuildToolsApi::class, ExperimentalKotlinGradlePluginApi::class) + +import org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi +import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi import org.jetbrains.kotlin.gradle.dsl.JvmTarget -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { `java-gradle-plugin` @@ -7,7 +10,6 @@ plugins { id("com.github.gmazzo.buildconfig") version "5.5.1" } -val kotlinCompilerVersion: String by project val kotlinPoetVersion: String by project repositories { @@ -17,11 +19,11 @@ repositories { buildConfig { packageName = "org.jetbrains.kotlinx.dataframe" className = "BuildConfig" - buildConfigField("kotlinCompilerVersion", kotlinCompilerVersion) + buildConfigField("kotlinCompilerVersion", kotlin.compilerVersion.get()) } dependencies { - compileOnly(kotlin("compiler-embeddable", kotlinCompilerVersion)) + compileOnly(kotlin("compiler-embeddable", kotlin.compilerVersion.get())) implementation("com.squareup:kotlinpoet:$kotlinPoetVersion") } diff --git a/plugins/keywords-generator/gradle.properties b/plugins/keywords-generator/gradle.properties index 7eade7a0da..079152fb53 100644 --- a/plugins/keywords-generator/gradle.properties +++ b/plugins/keywords-generator/gradle.properties @@ -1,2 +1 @@ -kotlinCompilerVersion=2.0.20 -kotlinPoetVersion=2.0.0 +kotlinPoetVersion=2.2.0 diff --git a/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorAction.kt b/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorAction.kt index e92f4bb240..b0baf5b049 100644 --- a/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorAction.kt +++ b/plugins/keywords-generator/src/main/kotlin/org/jetbrains/dataframe/keywords/KeywordsGeneratorAction.kt @@ -8,6 +8,7 @@ import com.squareup.kotlinpoet.TypeSpec import org.gradle.workers.WorkAction import org.gradle.workers.WorkParameters import org.jetbrains.kotlin.com.intellij.psi.tree.TokenSet +import org.jetbrains.kotlin.config.KotlinCompilerVersion import org.jetbrains.kotlin.lexer.KtKeywordToken import org.jetbrains.kotlin.lexer.KtTokens import java.io.File @@ -21,6 +22,7 @@ abstract class KeywordsGeneratorAction : WorkAction Date: Fri, 27 Jun 2025 16:12:01 +0200 Subject: [PATCH 5/9] fixed expressions-converter for Kotlin 2.2 --- .../dataframe/ExplainerIrTransformer.kt | 120 ++++++++++++------ .../AbstractExplainerBlackBoxCodegenTest.kt | 73 +++++------ 2 files changed, 115 insertions(+), 78 deletions(-) diff --git a/plugins/expressions-converter/src/org/jetbrains/kotlinx/dataframe/ExplainerIrTransformer.kt b/plugins/expressions-converter/src/org/jetbrains/kotlinx/dataframe/ExplainerIrTransformer.kt index 002a4c5bb6..75fed6cf06 100644 --- a/plugins/expressions-converter/src/org/jetbrains/kotlinx/dataframe/ExplainerIrTransformer.kt +++ b/plugins/expressions-converter/src/org/jetbrains/kotlinx/dataframe/ExplainerIrTransformer.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin import org.jetbrains.kotlin.ir.declarations.IrField import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.ir.declarations.IrParameterKind import org.jetbrains.kotlin.ir.declarations.path import org.jetbrains.kotlin.ir.expressions.IrBlockBody import org.jetbrains.kotlin.ir.expressions.IrBody @@ -23,7 +24,7 @@ import org.jetbrains.kotlin.ir.expressions.IrExpression import org.jetbrains.kotlin.ir.expressions.IrExpressionBody import org.jetbrains.kotlin.ir.expressions.IrGetValue import org.jetbrains.kotlin.ir.expressions.IrStatementOrigin -import org.jetbrains.kotlin.ir.expressions.impl.IrCallImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrCallImplWithShape import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl import org.jetbrains.kotlin.ir.expressions.impl.IrFunctionExpressionImpl import org.jetbrains.kotlin.ir.expressions.impl.IrGetObjectValueImpl @@ -38,8 +39,8 @@ import org.jetbrains.kotlin.ir.types.typeWith import org.jetbrains.kotlin.ir.util.SetDeclarationsParentVisitor import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable import org.jetbrains.kotlin.ir.util.isLocal -import org.jetbrains.kotlin.ir.visitors.IrElementTransformer import org.jetbrains.kotlin.ir.visitors.IrElementTransformerVoid +import org.jetbrains.kotlin.ir.visitors.IrTransformer import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid import org.jetbrains.kotlin.name.CallableId import org.jetbrains.kotlin.name.ClassId @@ -51,8 +52,8 @@ data class ContainingDeclarations(val clazz: IrClass?, val function: IrFunction? @OptIn(UnsafeDuringIrConstructionAPI::class) class ExplainerIrTransformer(val pluginContext: IrPluginContext) : - FileLoweringPass, - IrElementTransformer { + IrTransformer(), + FileLoweringPass { lateinit var file: IrFile lateinit var source: String @@ -151,12 +152,15 @@ class ExplainerIrTransformer(val pluginContext: IrPluginContext) : if (expression.startOffset < 0) return expression if (expression.type.classFqName in dataFrameLike) { if (expression.symbol.owner.name == Name.identifier("component1")) return expression - var receiver = expression.extensionReceiver - // expression.extensionReceiver = extension callables, + val extensionReceiverIndex = + expression.symbol.owner.parameters.indexOfFirst { it.kind == IrParameterKind.ExtensionReceiver } + var receiver: IrExpression? + // expression.arguments[extensionReceiverIndex] = extension callables, // expression.dispatchReceiver = member callables such as "GroupBy.aggregate" - if (receiver != null) { - val transformedExtensionReceiver = expression.extensionReceiver?.transform(this, data) - expression.extensionReceiver = transformedExtensionReceiver + if (extensionReceiverIndex >= 0) { + receiver = expression.arguments[extensionReceiverIndex]!! + val transformedExtensionReceiver = receiver.transform(this, data) + expression.arguments[extensionReceiverIndex] = transformedExtensionReceiver } else { receiver = expression.dispatchReceiver val transformedExtensionReceiver = expression.dispatchReceiver?.transform(this, data) @@ -179,9 +183,26 @@ class ExplainerIrTransformer(val pluginContext: IrPluginContext) : CallableId(FqName("kotlin"), Name.identifier("also")), ).single() - val result = IrCallImpl(-1, -1, expression.type, alsoReference, 1, 1).apply { - this.extensionReceiver = expression - putTypeArgument(0, expression.type) + val result = IrCallImplWithShape( + startOffset = -1, + endOffset = -1, + type = expression.type, + symbol = alsoReference, + typeArgumentsCount = 1, + valueArgumentsCount = 1, + contextParameterCount = 0, + hasDispatchReceiver = true, + hasExtensionReceiver = true, + ).apply { + val extensionReceiverIndex = + this.symbol.owner.parameters.indexOfFirst { it.kind == IrParameterKind.ExtensionReceiver } + if (extensionReceiverIndex >= 0) { + this.arguments[extensionReceiverIndex] = expression + } else { + this.insertExtensionReceiver(expression) + } + + typeArguments[0] = expression.type val symbol = IrSimpleFunctionSymbolImpl() val alsoLambda = pluginContext.irFactory @@ -202,25 +223,24 @@ class ExplainerIrTransformer(val pluginContext: IrPluginContext) : isInfix = false, isExpect = false, ).apply { - valueParameters = buildList { - add( - pluginContext.irFactory.createValueParameter( - startOffset = -1, - endOffset = -1, - origin = IrDeclarationOrigin.DEFINED, - symbol = IrValueParameterSymbolImpl(), - name = Name.identifier("it"), - index = 0, - type = expression.type, - varargElementType = null, - isCrossinline = false, - isNoinline = false, - isHidden = false, - isAssignable = false, - ), + // replace all regular value parameters with a single one `it` + parameters = parameters.filterNot { it.kind == IrParameterKind.Regular } + + pluginContext.irFactory.createValueParameter( + startOffset = -1, + endOffset = -1, + origin = IrDeclarationOrigin.DEFINED, + kind = IrParameterKind.Regular, + name = Name.identifier("it"), + type = expression.type, + isAssignable = false, + symbol = IrValueParameterSymbolImpl(), + varargElementType = null, + isCrossinline = false, + isNoinline = false, + isHidden = false, ) - } - val itSymbol = valueParameters[0].symbol + + val itSymbol = parameters.first { it.kind == IrParameterKind.Regular }.symbol val source = try { source.substring(expression.startOffset, expression.endOffset) } catch (e: Exception) { @@ -229,14 +249,21 @@ class ExplainerIrTransformer(val pluginContext: IrPluginContext) : val expressionId = expressionId(expression) val receiverId = receiver?.let { expressionId(it) } val valueArguments = buildList { - add(source.irConstImpl()) - add(ownerName.asStringStripSpecialMarkers().irConstImpl()) - add(IrGetValueImpl(-1, -1, itSymbol)) - add(expressionId.irConstImpl()) - add(receiverId.irConstImpl()) - add(data.clazz?.fqNameWhenAvailable?.asString().irConstImpl()) - add(data.function?.name?.asString().irConstImpl()) - add(IrConstImpl.int(-1, -1, pluginContext.irBuiltIns.intType, data.statementIndex)) + add(source.irConstImpl()) // source: String + add(ownerName.asStringStripSpecialMarkers().irConstImpl()) // name: String + add(IrGetValueImpl(-1, -1, itSymbol)) // df: Any + add(expressionId.irConstImpl()) // id: String + add(receiverId.irConstImpl()) // receiverId: String? + add(data.clazz?.fqNameWhenAvailable?.asString().irConstImpl()) // containingClassFqName: String? + add(data.function?.name?.asString().irConstImpl()) // containingFunName: String? + add( + IrConstImpl.int( + -1, + -1, + pluginContext.irBuiltIns.intType, + data.statementIndex, + ), + ) // statementIndex: Int } body = pluginContext.irFactory.createBlockBody(-1, -1).apply { val callableId = CallableId( @@ -245,19 +272,24 @@ class ExplainerIrTransformer(val pluginContext: IrPluginContext) : Name.identifier("doAction"), ) val doAction = pluginContext.referenceFunctions(callableId).single() - statements += IrCallImpl( + statements += IrCallImplWithShape( startOffset = -1, endOffset = -1, type = doAction.owner.returnType, symbol = doAction, typeArgumentsCount = 0, valueArgumentsCount = valueArguments.size, + contextParameterCount = 0, + hasDispatchReceiver = true, + hasExtensionReceiver = false, ).apply { val clazz = ClassId(explainerPackage, Name.identifier("PluginCallbackProxy")) val plugin = pluginContext.referenceClass(clazz)!! dispatchReceiver = IrGetObjectValueImpl(-1, -1, plugin.defaultType, plugin) + + val firstValueArgumentIndex = 1 // skipping dispatch receiver valueArguments.forEachIndexed { i, argument -> - putValueArgument(i, argument) + this.arguments[firstValueArgumentIndex + i] = argument } } } @@ -271,12 +303,16 @@ class ExplainerIrTransformer(val pluginContext: IrPluginContext) : function = alsoLambda, origin = IrStatementOrigin.LAMBDA, ) - putValueArgument(0, alsoLambdaExpression) + + val firstValueArgumentIndex = this.symbol.owner.parameters + .indexOfFirst { it.kind == IrParameterKind.Regular } + .takeUnless { it < 0 } ?: this.symbol.owner.parameters.size + this.arguments[firstValueArgumentIndex] = alsoLambdaExpression } return result } - private fun String?.irConstImpl(): IrConstImpl { + private fun String?.irConstImpl(): IrConstImpl { val nullableString = pluginContext.irBuiltIns.stringType.makeNullable() val argument = if (this == null) { IrConstImpl.constNull(-1, -1, nullableString) diff --git a/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/AbstractExplainerBlackBoxCodegenTest.kt b/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/AbstractExplainerBlackBoxCodegenTest.kt index 21684fd216..463517b780 100644 --- a/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/AbstractExplainerBlackBoxCodegenTest.kt +++ b/plugins/expressions-converter/tests/org/jetbrains/kotlinx/dataframe/AbstractExplainerBlackBoxCodegenTest.kt @@ -32,43 +32,44 @@ import java.io.File open class AbstractExplainerBlackBoxCodegenTest : BaseTestRunner() { - override fun TestConfigurationBuilder.configuration() { - globalDefaults { - frontend = FrontendKinds.ClassicAndFIR - targetPlatform = JvmPlatforms.jvm8 - dependencyKind = DependencyKind.Binary - targetBackend = TargetBackend.JVM_IR + override fun configure(builder: TestConfigurationBuilder): Unit = + with(builder) { + globalDefaults { + frontend = FrontendKinds.ClassicAndFIR + targetPlatform = JvmPlatforms.jvm8 + dependencyKind = DependencyKind.Binary + targetBackend = TargetBackend.JVM_IR + } + defaultDirectives { + JvmEnvironmentConfigurationDirectives.JDK_KIND with TestJdkKind.FULL_JDK + JvmEnvironmentConfigurationDirectives.JVM_TARGET with JvmTarget.JVM_1_8 + +JvmEnvironmentConfigurationDirectives.WITH_REFLECT + } + facadeStep(::ClassicFrontendFacade) + commonFirWithPluginFrontendConfiguration() + classicFrontendHandlersStep { + useHandlers( + ::ClassicDiagnosticsHandler, + ::DeclarationsDumpHandler, + ) + } + psi2IrStep() + irHandlersStep { + useHandlers( + ::IrPrettyKotlinDumpHandler, + ::IrTextDumpHandler, + ::IrTreeVerifierHandler, + ) + } + facadeStep(::JvmIrBackendFacade) + jvmArtifactsHandlersStep { + useHandlers(::JvmBoxRunner) + } + useConfigurators(::JvmEnvironmentConfigurator, ::CommonEnvironmentConfigurator, ::PluginAnnotationsProvider) + useCustomRuntimeClasspathProviders(::MyClasspathProvider) + useAfterAnalysisCheckers(::BlackBoxCodegenSuppressor) + useAdditionalService(::TemporaryDirectoryManagerImplFixed) } - defaultDirectives { - JvmEnvironmentConfigurationDirectives.JDK_KIND with TestJdkKind.FULL_JDK - JvmEnvironmentConfigurationDirectives.JVM_TARGET with JvmTarget.JVM_1_8 - +JvmEnvironmentConfigurationDirectives.WITH_REFLECT - } - facadeStep(::ClassicFrontendFacade) - commonFirWithPluginFrontendConfiguration() - classicFrontendHandlersStep { - useHandlers( - ::ClassicDiagnosticsHandler, - ::DeclarationsDumpHandler, - ) - } - psi2IrStep() - irHandlersStep { - useHandlers( - ::IrPrettyKotlinDumpHandler, - ::IrTextDumpHandler, - ::IrTreeVerifierHandler, - ) - } - facadeStep(::JvmIrBackendFacade) - jvmArtifactsHandlersStep { - useHandlers(::JvmBoxRunner) - } - useConfigurators(::JvmEnvironmentConfigurator, ::CommonEnvironmentConfigurator, ::PluginAnnotationsProvider) - useCustomRuntimeClasspathProviders(::MyClasspathProvider) - useAfterAnalysisCheckers(::BlackBoxCodegenSuppressor) - useAdditionalService(::TemporaryDirectoryManagerImplFixed) - } class MyClasspathProvider(testServices: TestServices) : RuntimeClasspathProvider(testServices) { override fun runtimeClassPaths(module: TestModule): List = From 7942a8b9e4dee81c4b501bf39364853c8d6b1b72 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Fri, 27 Jun 2025 16:22:04 +0200 Subject: [PATCH 6/9] disable korro and kodex at top-level due to leaking kotlin-compiler-embeddable --- build.gradle.kts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index c156c81955..8e8a8741e7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,8 +22,11 @@ plugins { alias(dokka) // alias(kover) alias(ktlint) - alias(korro) apply false - alias(kodex) apply false + + // TODO cannot define korro and kodex here due to leaking them kotlin-compiler-embeddable into the build classpath + // alias(korro) apply false + // alias(kodex) apply false + alias(simpleGit) apply false alias(dependencyVersions) alias(buildconfig) apply false From 40fe8decb9202498a64e3c94b402e47012a70b2d Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Tue, 1 Jul 2025 14:45:27 +0200 Subject: [PATCH 7/9] switched to dev.zacsweers.kctfork for k2 kotlin-compile-testing --- gradle/libs.versions.toml | 6 +++--- .../ksp/DataFrameSymbolProcessorTest.kt | 15 +++++---------- .../ksp/runner/KspCompilationTestRunner.kt | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8a723c9303..0dfff9bd26 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -53,7 +53,7 @@ plugin-publish = "1.3.0" shadow = "8.3.5" android-gradle-api = "7.3.1" # need to revise our tests to update ktor = "3.0.1" # needs jupyter compatibility with Kotlin 2.1 to update -kotlin-compile-testing = "1.6.0" +kotlin-compile-testing = "0.7.1" duckdb = "1.1.3" buildconfig = "5.6.7" benchmark = "0.4.12" @@ -152,8 +152,8 @@ android-gradle = { group = "com.android.tools.build", name = "gradle", version.r kotlin-gradle-plugin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin" } kotlin-gradle-plugin-api = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin-api" } ktor-server-netty = { group = "io.ktor", name = "ktor-server-netty", version.ref = "ktor" } -kotlin-compile-testing = { group = "com.github.tschuchortdev", name = "kotlin-compile-testing", version.ref = "kotlin-compile-testing" } -kotlin-compile-testing-ksp = { group = "com.github.tschuchortdev", name = "kotlin-compile-testing-ksp", version.ref = "kotlin-compile-testing" } +kotlin-compile-testing = { group = "dev.zacsweers.kctfork", name = "core", version.ref = "kotlin-compile-testing" } +kotlin-compile-testing-ksp = { group = "dev.zacsweers.kctfork", name = "ksp", version.ref = "kotlin-compile-testing" } kotlin-compiler = { group = "org.jetbrains.kotlin", name = "kotlin-compiler", version.ref = "kotlin" } kotlin-compiler-embeddable = { group = "org.jetbrains.kotlin", name = "kotlin-compiler-embeddable", version.ref = "kotlin" } kotlin-compiler-internal-test-framework = { group = "org.jetbrains.kotlin", name = "kotlin-compiler-internal-test-framework", version.ref = "kotlin" } diff --git a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt index f0c360d477..1c2d026079 100644 --- a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt +++ b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/DataFrameSymbolProcessorTest.kt @@ -1218,11 +1218,6 @@ class DataFrameSymbolProcessorTest { fun resolved() { JetBrains - JetBrains1 - JetBrains2 - JetBrains3 - JetBrains4 - JetBrains5 } """.trimIndent(), ), @@ -1268,11 +1263,11 @@ class DataFrameSymbolProcessorTest { ) println(result.kspGeneratedFiles) result.inspectLines("MetricsNoKeyValue.Generated.kt") { -// (('1'..'5') + "").forEach { nr -> -// it.forAtLeastOne { -// it shouldContain "JetBrains$nr" -// } -// } + (('1'..'3') + "").forEach { nr -> + it.forAtLeastOne { + it shouldContain "MetricsNoKeyValue$nr" + } + } } } diff --git a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt index ed673e2d18..a6757e0c5f 100644 --- a/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt +++ b/plugins/symbol-processor/src/test/kotlin/org/jetbrains/dataframe/ksp/runner/KspCompilationTestRunner.kt @@ -2,11 +2,13 @@ package org.jetbrains.dataframe.ksp.runner +import com.tschuchort.compiletesting.JvmCompilationResult import com.tschuchort.compiletesting.KotlinCompilation import com.tschuchort.compiletesting.SourceFile -import com.tschuchort.compiletesting.kspArgs +import com.tschuchort.compiletesting.kspProcessorOptions import com.tschuchort.compiletesting.kspSourcesDir import com.tschuchort.compiletesting.symbolProcessorProviders +import com.tschuchort.compiletesting.useKsp2 import org.jetbrains.dataframe.ksp.DataFrameSymbolProcessorProvider import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi import java.io.ByteArrayOutputStream @@ -15,7 +17,7 @@ import java.nio.file.Paths @Suppress("unused") internal class KotlinCompileTestingCompilationResult( - val delegate: KotlinCompilation.Result, + val delegate: JvmCompilationResult, val successfulCompilation: Boolean, val kspGeneratedFiles: List, val outputSourceDirs: List, @@ -43,8 +45,14 @@ internal object KspCompilationTestRunner { classpaths = params.classpath, tempDir = compilationDir, ) - kspCompilation.kspArgs.putAll(params.options) - kspCompilation.symbolProcessorProviders = listOf(DataFrameSymbolProcessorProvider()) + kspCompilation.kspProcessorOptions.putAll(params.options) + + // We don't support KSP2, but because we target Kotlin 2.2 the tests only work if I set them up like this. + kspCompilation.useKsp2() + kspCompilation.kspProcessorOptions["ksp.useKSP2"] = "false" + kspCompilation.kspProcessorOptions["useKSP2"] = "false" + + kspCompilation.symbolProcessorProviders = mutableListOf(DataFrameSymbolProcessorProvider()) kspCompilation.compile().also { println(it.messages) if (it.exitCode == KotlinCompilation.ExitCode.COMPILATION_ERROR) { From ef78df74cf0c90b59717ff66ea4faebca321cb6e Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Tue, 1 Jul 2025 15:54:42 +0200 Subject: [PATCH 8/9] disabling compiler plugin tests because we're moving it: #1290 --- plugins/kotlin-dataframe/build.gradle.kts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/kotlin-dataframe/build.gradle.kts b/plugins/kotlin-dataframe/build.gradle.kts index cdab139dc3..f07cc7463b 100644 --- a/plugins/kotlin-dataframe/build.gradle.kts +++ b/plugins/kotlin-dataframe/build.gradle.kts @@ -1,6 +1,5 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion -import org.jetbrains.kotlin.gradle.tasks.BaseKotlinCompile plugins { id("java") @@ -122,3 +121,13 @@ kotlinPublications { packageName = artifactId } } + +// Disabling all tests before removing the compiler plugin here +// because we're moving to the Kotlin repo: #1290 +tasks.filter { + ":plugins:kotlin-dataframe" in it.path && + "test" in it.name.lowercase() +}.forEach { + println("disabling compiler plugin test task: ${it.path}. See #1290") + it.onlyIf { false } +} From e1fc6a47d859ba316c628608d5e0720bfad605f2 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Tue, 1 Jul 2025 17:05:38 +0200 Subject: [PATCH 9/9] bumping jupyter version --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0dfff9bd26..935572d7c5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,6 @@ [versions] ksp = "2.2.0-2.0.2" -kotlinJupyter = "0.12.0-383" +kotlinJupyter = "0.13.0-481-1" ktlint = "12.3.0"