Add license report + NOTICE content check

NOTICE

@@ -6,3 +6,9 @@ The Apache Software Foundation (http://www.apache.org/).
 
 Apache Iceberg
 Copyright 2017-2022 The Apache Software Foundation
+
+---
+
+Polaris distributions contain some or all of the following dependencies.
+
+

aggregated-license-report/build.gradle.kts

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.gradle.kotlin.dsl.support.unzipTo
+
+val licenseReports by configurations.creating { description = "Used to reference license reports" }
+
+dependencies {
+  licenseReports(project(":polaris-service", "licenseReports"))
+}
+
+val collectLicenseReportJars by
+  tasks.registering(Sync::class) {
+    destinationDir = project.layout.buildDirectory.dir("tmp/license-report-jars").get().asFile
+    from(licenseReports)
+  }
+
+val aggregateLicenseReports by
+  tasks.registering {
+    group = "Build"
+    description = "Aggregates license reports"
+    val outputDir = project.layout.buildDirectory.dir("licenseReports")
+    outputs.dir(outputDir)
+    dependsOn(collectLicenseReportJars)
+    doLast {
+      delete(outputDir)
+      fileTree(collectLicenseReportJars.get().destinationDir).files.forEach { zip ->
+        val targetDirName = zip.name.replace("-license-report.zip", "")
+        unzipTo(outputDir.get().dir(targetDirName).asFile, zip)
+      }
+    }
+  }
+
+val aggregatedLicenseReportsZip by
+  tasks.registering(Zip::class) {
+    from(aggregateLicenseReports)
+    from(rootProject.layout.projectDirectory) {
+      include("NOTICE")
+      eachFile { path = "NOTICE.txt" }
+    }
+    archiveBaseName.set("polaris-aggregated-license-report-${project.version}")
+    destinationDirectory.set(layout.buildDirectory.dir("distributions"))
+    archiveExtension.set("zip")
+  }

build-logic/build.gradle.kts

@@ -20,5 +20,6 @@ plugins { `kotlin-dsl` }
 dependencies {
   implementation(gradleKotlinDsl())
   implementation(baselibs.idea.ext)
+  implementation(baselibs.license.report)
   implementation(baselibs.spotless)
 }

build-logic/src/main/kotlin/NoticeReportValidation.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.github.jk1.license.LicenseReportExtension
+import com.github.jk1.license.ProjectData
+import com.github.jk1.license.filter.DependencyFilter
+import java.io.File
+import org.gradle.api.GradleException
+
+/**
+ * Validates that all dependencies with MIT/BSD/Go/UPL/ISC licenses, which do not have an Apache
+ * license, are mentioned in the `NOTICE` file.
+ */
+class NoticeReportValidation : DependencyFilter {
+  fun needsNoNotice(license: String?): Boolean = license != null && (license.contains("Apache"))
+
+  fun needsNotice(license: String?): Boolean =
+    license != null &&
+      (license.contains("MIT") ||
+        license.contains("BSD") ||
+        license.contains("Go") ||
+        license.contains("ISC") ||
+        license.contains("Universal Permissive"))
+
+  override fun filter(data: ProjectData?): ProjectData {
+    data!!
+
+    val rootNoticeFile = data.project.rootProject.file("NOTICE").readText()
+
+    val licenseReport = data.project.extensions.getByType(LicenseReportExtension::class.java)
+
+    val missing = mutableMapOf<String, String>()
+
+    data.allDependencies.forEach { mod ->
+      val licenses =
+        (mod.manifests.map { it.license } +
+            mod.licenseFiles.flatMap { it.fileDetails }.map { it.license } +
+            mod.poms.flatMap { it.licenses }.map { it.name })
+          .distinct()
+
+      if (!licenses.any { needsNoNotice(it) } && licenses.any { needsNotice(it) }) {
+        val groupModule = "${mod.group}:${mod.name}"
+        if (!rootNoticeFile.contains(groupModule)) {
+          missing.put(
+            "${mod.group}:${mod.name}",
+            """
+            ---
+            ${mod.group}:${mod.name}
+
+            ${mod.licenseFiles.flatMap { it.fileDetails }.filter { it.file != null }.map { it.file }
+              .map { File("${licenseReport.absoluteOutputDir}/$it").readText().trim() }
+              .distinct()
+              .map { "\n\n$it\n" }
+              .joinToString("\n")
+            }
+            """
+              .trimIndent()
+          )
+        }
+      }
+    }
+
+    if (!missing.isEmpty()) {
+      throw GradleException(
+        "License information for the following artifacts is missing in the root NOTICE file: ${missing.map { it.value }.joinToString("\n")}"
+      )
+    }
+
+    return data
+  }
+}

build-logic/src/main/kotlin/polaris-license-report.gradle.kts

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.github.jk1.license.filter.LicenseBundleNormalizer
+import com.github.jk1.license.render.InventoryHtmlReportRenderer
+import com.github.jk1.license.render.JsonReportRenderer
+import com.github.jk1.license.render.XmlReportRenderer
+import java.util.*
+
+plugins { id("com.github.jk1.dependency-license-report") }
+
+afterEvaluate {
+  // Need to configure after evaluation, because the spark-extensions project use a custom
+  // `buildDir`.
+  licenseReport {
+    filters =
+      arrayOf(
+        LicenseBundleNormalizer(
+          "${rootProject.projectDir}/gradle/license/normalizer-bundle.json",
+          false
+        ),
+        NoticeReportValidation()
+      )
+    allowedLicensesFile = rootProject.projectDir.resolve("gradle/license/allowed-licenses.json")
+    renderers =
+      arrayOf(InventoryHtmlReportRenderer("index.html"), JsonReportRenderer(), XmlReportRenderer())
+    excludeBoms = true
+    excludes =
+      arrayOf(
+        "com.google.guava:guava-parent",
+        "io.opentelemetry:opentelemetry-bom-alpha",
+        "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha"
+      )
+    outputDir = "${project.layout.buildDirectory.get()}/reports/dependency-license"
+    excludeGroups = arrayOf("org.projectnessie.nessie", "org.projectnessie.nessie-integrations")
+  }
+}
+
+val generateLicenseReport =
+  tasks.named("generateLicenseReport") {
+    inputs
+      .files(
+        rootProject.projectDir.resolve("gradle/license/normalizer-bundle.json"),
+        rootProject.projectDir.resolve("gradle/license/allowed-licenses.json")
+      )
+      .withPathSensitivity(PathSensitivity.RELATIVE)
+    inputs.property("renderersHash", Arrays.hashCode(licenseReport.renderers))
+    inputs.property("filtersHash", Arrays.hashCode(licenseReport.filters))
+    inputs.property("excludesHash", Arrays.hashCode(licenseReport.excludes))
+    inputs.property("excludeGroupsHash", Arrays.hashCode(licenseReport.excludeGroups))
+  }
+
+val licenseReportZip =
+  tasks.register<Zip>("licenseReportZip") {
+    group = "documentation"
+    description = "License report as a ZIP"
+    dependsOn("checkLicense")
+    from(generateLicenseReport)
+    archiveClassifier.set("license-report")
+    archiveExtension.set("zip")
+  }
+
+val licenseReports by
+  configurations.creating {
+    isCanBeConsumed = true
+    isCanBeResolved = false
+    description = "License report files"
+    outgoing { artifact(licenseReportZip) }
+  }
+
+plugins.withType<MavenPublishPlugin>().configureEach {
+  configure<PublishingExtension> {
+    publications { named<MavenPublication>("maven") { artifact(licenseReportZip) } }
+  }
+}
+
+tasks.named("check") { dependsOn(generateLicenseReport) }

gradle/baselibs.versions.toml

@@ -14,4 +14,5 @@
 
 [libraries]
 idea-ext = { module = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext", version = "1.1.8" }
+license-report = { module = "com.github.jk1:gradle-license-report", version = "2.8" }
 spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = "6.25.0" }

gradle/license/allowed-licenses.json

@@ -0,0 +1,96 @@
+{
+  "allowedLicenses": [
+    {
+      "moduleLicense": "Apache Software License, Version 2.0"
+    },
+    {
+      "moduleLicense": "Apache Software License, version 2.0"
+    },
+    {
+      "moduleLicense": "Apache License, Version 2.0"
+    },
+    {
+      "moduleLicense": "Apache 2"
+    },
+    {
+      "moduleLicense": "Apache 2.0"
+    },
+    {
+      "moduleLicense": "Apache-2.0"
+    },
+    {
+      "moduleLicense": "BSD-2.Clause"
+    },
+    {
+      "moduleLicense": "Revised BSD"
+    },
+    {
+      "moduleLicense": "BSD-3.Clause"
+    },
+    {
+      "moduleLicense": "BSD New license"
+    },
+    {
+      "moduleLicense": "Common Development and Distribution License (CDDL) v1.0"
+    },
+    {
+      "moduleLicense": "Creative Commons 1.0 Universal"
+    },
+    {
+      "moduleLicense": "Eclipse Distribution License, Version 1.0"
+    },
+    {
+      "moduleLicense": "EDL 1.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License, Version 1.0"
+    },
+    {
+      "moduleLicense": "EPL 2.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License, Version 2.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License - v 2.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License v. 2.0"
+    },
+    {
+      "moduleLicense": "GNU General Public License, Version 2 with the GNU Classpath Exception"
+    },
+    {
+      "moduleLicense": "GNU Lesser General Public License Version 2.1"
+    },
+    {
+      "moduleLicense": "Go License"
+    },
+    {
+      "moduleLicense": "MIT License"
+    },
+    {
+      "moduleLicense": "MIT-0"
+    },
+    {
+      "moduleLicense": "MIT"
+    },
+    {
+      "moduleLicense": "Public Domain"
+    },
+    {
+      "moduleLicense": "Public Domain, per Creative Commons CC0"
+    },
+    {
+      "moduleLicense": "Universal Permissive License, Version 1.0"
+    },
+    {
+      "moduleLicense": "Bouncy Castle Licence"
+    },
+    {
+      "moduleLicense": null,
+      "moduleName": "javax.servlet.jsp:jsp-api",
+      "moduleVersion": "2.1"
+    }
+  ]
+}