Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support configuration cache #206

Merged
merged 31 commits into from
Sep 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
be86a6e
Move filtering of proto dirs to the caller
sebek64 Aug 23, 2024
60f9e16
Move filtering of proto dirs to the caller
sebek64 Aug 23, 2024
1ac6665
Move filtering of proto dirs to the caller
sebek64 Aug 23, 2024
a9e94fc
Move filtering of proto dirs to the caller
sebek64 Aug 23, 2024
e2e10f4
Move filtering of proto dirs to the caller
sebek64 Aug 23, 2024
0636c97
Inherit all buf-executing tasks from a new AbstractBufExecTask
sebek64 Aug 23, 2024
68e5081
Rename registerBufTask -> registerBufExecTask
sebek64 Aug 23, 2024
f0d369c
Introduce empty AbstractBufTask and inherit all tasks from it
sebek64 Aug 23, 2024
9446634
Introduce a function for creating buf tasks
sebek64 Aug 23, 2024
8495a77
Extend only AbstractBufExecTask by execution functions, not a Task
sebek64 Aug 23, 2024
155412d
Extend a concrete task
sebek64 Aug 23, 2024
8f16dc9
Move mangling functions from project to abstract task
sebek64 Aug 23, 2024
3ee2f49
Do not refer to project in mangling - pass the value as task input
sebek64 Aug 23, 2024
13294df
Make CreateSymLinksToModulesTask config-cache compatible
sebek64 Aug 23, 2024
2d2fbb9
Make WriteWorkspaceYamlTask config cache compatible
sebek64 Jul 2, 2024
04437d0
Remove no longer needed createsOutput call - outputs now created auto…
sebek64 Jul 3, 2024
a9bd131
Look up buf executable in configuration phase and pass it as tasks input
sebek64 Jul 2, 2024
5383812
Pass hasWorkspace as task input
sebek64 Jul 2, 2024
76623b6
Pass hasProtobufGradlePlugin as task input
sebek64 Jul 1, 2024
c79e12a
Pass candidateProtoDirs as task input if needed
sebek64 Jul 1, 2024
8139707
Pass workingDir as task input
sebek64 Jul 2, 2024
ffa36c4
Remove no longer needed with block
sebek64 Jul 2, 2024
9efbf8e
Fix BreakingTask to support config cache
sebek64 Jul 2, 2024
a450ff4
Fix BuildTask to support config cache
sebek64 Jul 2, 2024
e74ff1a
Fix GenerateTask to support config cache
sebek64 Jul 2, 2024
51ffd67
Fix format tasks to have correct inputs / outputs
sebek64 Jul 3, 2024
82ff799
Do not lookup buf config file in the lint task
sebek64 Jul 2, 2024
5d31398
Remove unnecessary and misleading extension functions
sebek64 Jul 3, 2024
c83739a
Enable configuration cache in tests
sebek64 Jul 2, 2024
24091e2
Merge branch 'main' into support-configuration-cache
andrewparmet Aug 26, 2024
f300616
Merge branch 'main' into support-configuration-cache
andrewparmet Aug 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions src/main/kotlin/build/buf/gradle/AbstractBufExecTask.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright 2024 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build.buf.gradle

import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import java.io.File

/**
* A task executing buf executable as part of its operation.
*/
abstract class AbstractBufExecTask : AbstractBufTask() {
/** The buf executable. */
@get:InputFiles
internal abstract val bufExecutable: ConfigurableFileCollection

/**
* The directory in which buf is executed.
* The actual real input files in this directory have to be tracked per command separately,
* so it is just an @Input, not @InputDirectory.
*/
@get:Input
internal abstract val workingDir: Property<File>

/** Whether the project has protobuf plugin enabled. */
@get:Input
internal abstract val hasProtobufGradlePlugin: Property<Boolean>

/** Directories possibly containing input .proto files. */
@get:InputFiles
internal abstract val candidateProtoDirs: ConfigurableFileCollection

/** Whether the project has buf workspace or not. */
@get:Input
internal abstract val hasWorkspace: Property<Boolean>
}
29 changes: 29 additions & 0 deletions src/main/kotlin/build/buf/gradle/AbstractBufTask.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright 2024 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build.buf.gradle

import org.gradle.api.DefaultTask
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import java.io.File

abstract class AbstractBufTask : DefaultTask() {
/**
* This property has to be set to project directory. It is only used for relativization of paths,
* so it is just an @Input, not @InputDirectory.
*/
@get:Input
internal abstract val projectDir: Property<File>
}
10 changes: 9 additions & 1 deletion src/main/kotlin/build/buf/gradle/BreakingConfiguration.kt
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,18 @@ private fun Project.addSchemaDependency(artifactDetails: ArtifactDetails) {
}

private fun Project.configureBreakingTask() {
registerBufTask<BreakingTask>(BUF_BREAKING_TASK_NAME) {
registerBufExecTask<BreakingTask>(BUF_BREAKING_TASK_NAME) {
group = VERIFICATION_GROUP
description = "Checks that Protobuf API definitions are backwards-compatible with previous versions."

dependsOn(BUF_BUILD_TASK_NAME)

if (project.bufV1SyntaxOnly()) {
v1SyntaxOnly.set(true)
publicationFile.set(project.bufBuildPublicationFile)
} else {
v1SyntaxOnly.set(false)
}
configFile.set(singleFileFromConfiguration(BUF_BREAKING_CONFIGURATION_NAME))
}
}
26 changes: 21 additions & 5 deletions src/main/kotlin/build/buf/gradle/BreakingTask.kt
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,35 @@

package build.buf.gradle

import org.gradle.api.DefaultTask
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import java.io.File

abstract class BreakingTask : AbstractBufExecTask() {
@get:Input
internal abstract val v1SyntaxOnly: Property<Boolean>

/** The input publication file. */
@get:InputFile
@get:Optional
internal abstract val publicationFile: Property<File>

/** The input breaking config file. */
@get:InputFile
internal abstract val configFile: Property<File>

abstract class BreakingTask : DefaultTask() {
@TaskAction
fun bufBreaking() {
val args = mutableListOf<Any>()
args.add("breaking")
if (project.bufV1SyntaxOnly()) {
args.add(bufBuildPublicationFile)
if (v1SyntaxOnly.get()) {
args.add(publicationFile.get())
}
args.add("--against")
args.add(singleFileFromConfiguration(BUF_BREAKING_CONFIGURATION_NAME))
args.add(configFile.get())
execBuf(*args.toTypedArray()) {
"""
|Some Protobuf files had breaking changes:
Expand Down
1 change: 1 addition & 0 deletions src/main/kotlin/build/buf/gradle/BufPlugin.kt
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class BufPlugin : Plugin<Project> {
}

private fun Project.configureBuf() {
createBufBinaryDependencyConfiguration()
configureLint()
configureFormat()
configureBuild()
Expand Down
83 changes: 44 additions & 39 deletions src/main/kotlin/build/buf/gradle/BufSupport.kt
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@
package build.buf.gradle

import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.kotlin.dsl.dependencies
import java.nio.charset.StandardCharsets

const val BUF_BINARY_CONFIGURATION_NAME = "bufTool"

internal fun Project.createBufBinaryDependencyConfiguration() {
configurations.create(BUF_BINARY_CONFIGURATION_NAME)
}

internal fun Project.configureBufDependency() {
val os = System.getProperty("os.name").lowercase()
val osPart =
Expand All @@ -39,60 +43,61 @@ internal fun Project.configureBufDependency() {

val extension = getExtension()

createConfigurationWithDependency(
BUF_BINARY_CONFIGURATION_NAME,
mapOf(
"group" to "build.buf",
"name" to "buf",
"version" to extension.toolVersion,
"classifier" to "$osPart-$archPart",
"ext" to "exe",
),
)
dependencies {
add(
BUF_BINARY_CONFIGURATION_NAME,
mapOf(
"group" to "build.buf",
"name" to "buf",
"version" to extension.toolVersion,
"classifier" to "$osPart-$archPart",
"ext" to "exe",
),
)
}
}

internal fun Task.execBuf(
internal fun AbstractBufExecTask.execBuf(
vararg args: Any,
customErrorMessage: ((String) -> String)? = null,
) {
execBuf(args.asList(), customErrorMessage)
}

internal fun Task.execBuf(
internal fun AbstractBufExecTask.execBuf(
args: Iterable<Any>,
customErrorMessage: ((String) -> String)? = null,
) {
with(project) {
val executable = singleFileFromConfiguration(BUF_BINARY_CONFIGURATION_NAME)

if (!executable.canExecute()) {
executable.setExecutable(true)
}
val executable = bufExecutable.singleFile

val workingDir =
if (hasProtobufGradlePlugin()) {
bufbuildDir
} else {
projectDir
}
if (!executable.canExecute()) {
executable.setExecutable(true)
}

val processArgs = listOf(executable.absolutePath) + args
val processArgs = listOf(executable.absolutePath) + args
val workingDirValue = workingDir.get()

logger.info("Running buf from $workingDir: `buf ${args.joinToString(" ")}`")
val result = ProcessRunner().use { it.shell(workingDir, processArgs) }
logger.info("Running buf from $workingDirValue: `buf ${args.joinToString(" ")}`")
val result = ProcessRunner().use { it.shell(workingDirValue, processArgs) }

if (result.exitCode != 0) {
if (customErrorMessage != null) {
val stdOut = result.stdOut.toString(StandardCharsets.UTF_8)
val stdErr = result.stdErr.toString(StandardCharsets.UTF_8)
val ex = IllegalStateException(customErrorMessage(stdOut))
if (stdErr.isNotEmpty()) {
ex.addSuppressed(IllegalStateException(result.toString()))
}
throw ex
} else {
error(result.toString())
if (result.exitCode != 0) {
if (customErrorMessage != null) {
val stdOut = result.stdOut.toString(StandardCharsets.UTF_8)
val stdErr = result.stdErr.toString(StandardCharsets.UTF_8)
val ex = IllegalStateException(customErrorMessage(stdOut))
if (stdErr.isNotEmpty()) {
ex.addSuppressed(IllegalStateException(result.toString()))
}
throw ex
} else {
error(result.toString())
}
}
}

internal fun AbstractBufExecTask.obtainDefaultProtoFileSet() =
project.fileTree(workingDir.get()) {
include("**/*.proto")
// not to interfere with random plugins producing output to build dir
exclude("build")
}
14 changes: 5 additions & 9 deletions src/main/kotlin/build/buf/gradle/BuildConfiguration.kt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package build.buf.gradle

import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.publish.PublishingExtension
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.kotlin.dsl.create
Expand All @@ -28,16 +27,16 @@ private const val BUF_BUILD_PUBLICATION_FILE_BASE_NAME = "image"
const val BUF_IMAGE_PUBLICATION_NAME = "bufImagePublication"

internal fun Project.configureBuild() {
registerBufTask<BuildTask>(BUF_BUILD_TASK_NAME) {
registerBufExecTask<BuildTask>(BUF_BUILD_TASK_NAME) {
group = BUILD_GROUP
description = "Builds a Buf image from a Protobuf schema."

if (hasProtobufGradlePlugin()) {
dependsOn(COPY_BUF_CONFIG_TASK_NAME)
} else {
// Called already during workspace configuration if the protobuf-gradle-plugin has been applied
createsOutput()
}

inputFiles.setFrom(obtainDefaultProtoFileSet())
publicationFile.set(project.bufBuildPublicationFile)
}
}

Expand All @@ -64,8 +63,5 @@ private val Project.bufBuildPublicationFileExtension
deets.imageFormat.formatName + deets.compressionFormat?.let { ".${it.ext}" }.orEmpty()
}

private val Project.bufBuildPublicationFile
internal val Project.bufBuildPublicationFile
get() = File(bufbuildDir, "$BUF_BUILD_PUBLICATION_FILE_BASE_NAME.$bufBuildPublicationFileExtension")

internal val Task.bufBuildPublicationFile
get() = project.bufBuildPublicationFile
18 changes: 15 additions & 3 deletions src/main/kotlin/build/buf/gradle/BuildTask.kt
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,24 @@

package build.buf.gradle

import org.gradle.api.DefaultTask
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.provider.Property
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import java.io.File

abstract class BuildTask : AbstractBufExecTask() {
/** The input files. */
@get:InputFiles
internal abstract val inputFiles: ConfigurableFileCollection

/** The output publication file. */
@get:OutputFile
internal abstract val publicationFile: Property<File>

abstract class BuildTask : DefaultTask() {
@TaskAction
fun bufBuild() {
execBuf("build", "--output", bufBuildPublicationFile)
execBuf("build", "--output", publicationFile.get())
}
}
5 changes: 1 addition & 4 deletions src/main/kotlin/build/buf/gradle/ConfigSupport.kt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package build.buf.gradle

import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.Copy
import org.gradle.kotlin.dsl.register
import java.io.File
Expand All @@ -25,7 +24,7 @@ const val COPY_BUF_CONFIG_TASK_NAME = "copyBufConfig"
internal fun Project.configureCopyBufConfig() {
tasks.register<Copy>(COPY_BUF_CONFIG_TASK_NAME) {
from(listOfNotNull(bufConfigFile()))
into(bufbuildDir)
into(project.bufbuildDir)
rename { "buf.yaml" }
}
}
Expand All @@ -47,8 +46,6 @@ internal fun Project.bufConfigFile() =
}
}

internal fun Task.bufConfigFile() = project.bufConfigFile()

private fun Project.resolveConfig(): File? {
val ext = getExtension()
return configurations.getByName(BUF_CONFIGURATION_NAME).let {
Expand Down
Loading