Skip to content

[Build] [DO NOT MERGE YET] Refactor validation-jar checks #14861

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

[Build] [DO NOT MERGE YET] Refactor validation-jar checks #14861

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e9b5b28
[Build] Refactor validation-jar checks
breskeby Jun 28, 2025
bb198ef
Extract jarInfo generation into binary task implementation
breskeby Jun 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build-tools/build-infra-shadow/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ dependencies {
implementation deps.randomizedtesting.runner
implementation deps.gjf
implementation deps.jgit
implementation deps.gson

implementation plugin(deps.plugins.carrotsearch.buildopts)
implementation plugin(deps.plugins.carrotsearch.dependencychecks)
Expand Down
1 change: 1 addition & 0 deletions build-tools/build-infra/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ dependencies {
implementation deps.flexmark.ext.tables
implementation deps.gjf
implementation deps.jgit
implementation deps.gson

implementation plugin(deps.plugins.carrotsearch.buildopts)
implementation plugin(deps.plugins.carrotsearch.dependencychecks)
Expand Down
177 changes: 14 additions & 163 deletions build-tools/build-infra/src/main/groovy/lucene.validation.jar-checks.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
*/

import org.apache.commons.codec.digest.DigestUtils
import org.apache.lucene.gradle.validation.JarInfo
import org.apache.lucene.gradle.validation.JarValidationPlugin
import org.apache.lucene.gradle.validation.JarValidationTask

// This adds validation of project dependencies:
// 1) license file
Expand All @@ -40,25 +43,6 @@ boolean failOnError = true
// relative to the current project so they're not the same).
def licensesDir = rootProject.layout.projectDirectory.dir("lucene/licenses")


// All known license types. If 'noticeOptional' is true then
// the notice file must accompany the license.
def licenseTypes = [
"ASL" : [name: "Apache Software License 2.0"],
"BSD" : [name: "Berkeley Software Distribution"],
//BSD like just means someone has taken the BSD license and put in their name, copyright, or it's a very similar license.
"BSD_LIKE": [name: "BSD like license"],
"CDDL" : [name: "Common Development and Distribution License", noticeOptional: true],
"CPL" : [name: "Common Public License"],
"EPL" : [name: "Eclipse Public License Version 1.0", noticeOptional: true],
"MIT" : [name: "Massachusetts Institute of Tech. License", noticeOptional: true],
"MPL" : [name: "Mozilla Public License", noticeOptional: true /* NOT SURE on the required notice */],
"PD" : [name: "Public Domain", noticeOptional: true],
"PDDL" : [name: "Public Domain Dedication and License", noticeOptional: true],
"SUN" : [name: "Sun Open Source License", noticeOptional: true],
"COMPOUND": [name: "Compound license (details in NOTICE file)."],
]

allprojects {
def licensesTask = tasks.register("licenses", {
group = 'Dependency validation'
Expand All @@ -71,18 +55,14 @@ allprojects {
}

subprojects {
// initialize empty, because no checks for benchmark-jmh module.
ext.jarInfos = []

apply plugin: JarValidationPlugin
// Configure jarValidation configuration for all projects. Any dependency
// declared on this configuration (or any configuration it extends from) will
// be verified.
configurations {
jarValidation
}

// For Java projects, add all dependencies from the following configurations
// to jar validation
plugins.withType(JavaPlugin).configureEach {
configurations {
jarValidation {
Expand All @@ -94,75 +74,6 @@ subprojects {
}
}

// Collects dependency JAR information for a project and saves it in
// project.ext.jarInfos. Each dependency has a map of attributes
// which make it easier to process it later on (name, hash, origin module,
// see the code below for details).
def collectJarInfos = tasks.register("collectJarInfos", {
dependsOn configurations.jarValidation

doFirst {
// When gradle resolves a configuration it applies exclude rules from inherited configurations
// globally (this seems like a bug to me). So we process each inherited configuration independently
// but make sure there are no other dependencies on jarValidation itself.
if (!configurations.jarValidation.dependencies.isEmpty()) {
throw new GradleException("jarValidation must only inherit from other configurations (can't have its own dependencies).")
}

def excludeRules = configurations.jarValidation.excludeRules

ArrayDeque<ResolvedDependency> queue = new ArrayDeque<>()
configurations.jarValidation.extendsFrom.each { conf ->
if (excludeRules) {
conf = conf.copyRecursive()
conf.canBeResolved = true
conf.canBeConsumed = true
conf.excludeRules = excludeRules
}
if (conf.canBeResolved) {
queue.addAll(conf.resolvedConfiguration.firstLevelModuleDependencies)
}
}

def visited = new HashSet<>()
def seenDeps = new HashSet<>()
def infos = []

while (!queue.isEmpty()) {
def dep = queue.removeFirst()
seenDeps.add(dep)

// Skip any artifacts from Lucene modules.
if (!dep.moduleGroup.startsWith("org.apache.lucene")) {
// Make sure we don't keep visiting the same children over and over again
dep.children.each { child ->
if (!seenDeps.contains(child)) {
queue.add(child)
}
}
def digestUtils = new DigestUtils(DigestUtils.getSha1Digest())
dep.moduleArtifacts.each { resolvedArtifact ->
def file = resolvedArtifact.file
if (visited.add(file)) {
infos.add([
name : resolvedArtifact.name,
jarName : file.toPath().getFileName().toString(),
path : file,
module : resolvedArtifact.moduleVersion,
checksum : provider { digestUtils.digestAsHex(file).trim() },
// We keep track of the files referenced by this dependency (sha, license, notice, etc.)
// so that we can determine unused dangling files later on.
referencedFiles: []
])
}
}
}
}

project.ext.jarInfos = infos.sort {a, b -> "${a.module}".compareTo("${b.module}")}
}
})

// Verifies that each JAR has a corresponding checksum and that it matches actual JAR available for this dependency.
tasks.register("validateJarChecksums", {
group = 'Dependency validation'
Expand All @@ -172,14 +83,14 @@ subprojects {

doLast {
def errors = []
project.ext.jarInfos.each { dep ->
project.jarInfos.each {dep ->
def expectedChecksumFile = licensesDir.file("${dep.jarName}.sha1").asFile
if (!expectedChecksumFile.exists()) {
errors << "Dependency checksum missing ('${dep.module}'), expected it at: ${expectedChecksumFile}"
} else {
dep.referencedFiles += expectedChecksumFile
dep.addReferencedFile(expectedChecksumFile)
def expected = expectedChecksumFile.getText("UTF-8").trim()
def actual = dep.checksum.get()
def actual = dep.checksum
if (expected.compareToIgnoreCase(actual) != 0) {
errors << "Dependency checksum mismatch ('${dep.module}'), expected it to be: ${expected}, but was: ${actual}"
} else {
Expand All @@ -204,67 +115,9 @@ subprojects {
// where 'jar-or-prefix' can be any '-'-delimited prefix of the dependency JAR's name.
// So for 'commons-io' it can be 'commons-io-LICENSE-foo.txt' or
// 'commons-LICENSE.txt'
tasks.register("validateJarLicenses", {
group = 'Dependency validation'
description = "Validate license and notice files of dependencies"
tasks.named("validateJarLicenses").configure {
dependsOn "collectJarInfos"

doLast {
def errors = []
project.ext.jarInfos.each { dep ->
def baseName = dep.name
def found = []
def candidates = []
while (true) {
candidates += licensesDir.file("${baseName}-LICENSE-[type].txt").asFile
found += fileTree(dir: licensesDir, include: "${baseName}-LICENSE-*.txt").files
def prefix = baseName.replaceAll(/[\-][^-]+$/, "")
if (found || prefix == baseName) {
break
}
baseName = prefix
}

if (found.size() == 0) {
errors << "License file missing ('${dep.module}'), expected it at: ${candidates.join(" or ")}," +
" where [type] can be any of ${licenseTypes.keySet()}."
} else if (found.size() > 1) {
errors << "Multiple license files matching for ('${dep.module}'): ${found.join(", ")}"
} else {
def licenseFile = found.get(0)
dep.referencedFiles += licenseFile
def m = (licenseFile.name =~ /LICENSE-(.+)\.txt$/)
if (!m) throw new GradleException("License file name doesn't contain license type?: ${licenseFile.name}")

def licenseName = m[0][1]
def licenseType = licenseTypes[licenseName]
if (!licenseType) {
errors << "Unknown license type suffix for ('${dep.module}'): ${licenseFile} (must be one of ${licenseTypes.keySet()})"
} else {
logger.log(LogLevel.INFO, "Dependency license file OK ('${dep.module}'): " + licenseName)

// Look for sibling NOTICE file.
def noticeFile = file(licenseFile.path.replaceAll(/\-LICENSE-.+/, "-NOTICE.txt"))
if (noticeFile.exists()) {
dep.referencedFiles += noticeFile
logger.log(LogLevel.INFO, "Dependency notice file OK ('${dep.module}'): " + noticeFile)
} else if (!licenseType.noticeOptional) {
errors << "Notice file missing for ('${dep.module}'), expected it at: ${noticeFile}"
}
}
}
}

if (errors) {
def msg = "Certain license/ notice files are missing:\n - " + errors.join("\n - ")
if (failOnError) {
throw new GradleException(msg)
} else {
logger.log(LogLevel.WARN, "WARNING: ${msg}")
}
}
}
})
}

tasks.named("licenses").configure {
dependsOn "validateJarChecksums", "validateJarLicenses"
Expand All @@ -283,9 +136,7 @@ configure(project(":lucene")) {
]

def validationTasks = subprojects.collectMany {
it.tasks.matching {
it.name == "licenses"
}
it.tasks.withType(JarValidationTask.class)
}
def jarInfoTasks = subprojects.collectMany {
it.tasks.matching {
Expand All @@ -312,9 +163,9 @@ configure(project(":lucene")) {
})

def updated = []
jarInfoTasks.collectMany { task -> task.project.ext.jarInfos }.each { dep ->
jarInfoTasks.collectMany {task -> task.project.jarInfos}.each {dep ->
def expectedChecksumFile = file("${licensesDir}/${dep.jarName}.sha1")
def actual = dep.checksum.get()
def actual = dep.checksum
if (expectedChecksumFile.exists()) {
def expected = expectedChecksumFile.getText("UTF-8").trim()
if (expected.compareToIgnoreCase(actual) == 0) {
Expand All @@ -326,7 +177,7 @@ configure(project(":lucene")) {
expectedChecksumFile.write(actual + "\n", "UTF-8")
}

updated.sort().each { line -> logger.log(LogLevel.LIFECYCLE, line) }
updated.sort().each {line -> logger.log(LogLevel.LIFECYCLE, line)}
}
})

Expand Down Expand Up @@ -365,7 +216,7 @@ configure(project(":lucene")) {
// Only collect for enabled tasks: https://issues.apache.org/jira/browse/LUCENE-9780
.findAll { it.enabled }
.collectMany { task ->
task.project.ext.jarInfos.collectMany { it.referencedFiles }
task.jarInfos.collectMany { it.referencedFiles }
}
.collect { it.toString() }

Expand Down
Loading
Loading