From 300f7ccfef801d1b816cb70f5230d87ab61d38d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Sep 2024 16:39:47 +0200 Subject: [PATCH 01/36] Fix missing parts in LICENSE (#266) --- LICENSE | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/LICENSE b/LICENSE index 3060ae578..799d745cf 100644 --- a/LICENSE +++ b/LICENSE @@ -236,6 +236,7 @@ com.google.api:api-common com.google.api:gax com.google.api:gax-grpc com.google.api:gax-httpjson +com.google.api.grpc:proto-google-cloud-monitoring-v3 com.google.apis:google-api-services-storage com.google.auth:google-auth-library-credentials com.google.auth:google-auth-library-oauth2-http @@ -244,6 +245,11 @@ com.google.cloud:google-cloud-core com.google.cloud:google-cloud-core-grpc com.google.cloud:google-cloud-core-http com.google.cloud:google-cloud-storage +com.google.cloud:google-cloud-monitoring +com.google.cloud.opentelemetry:detector-resources-support +com.google.cloud.opentelemetry:exporter-metrics +com.google.cloud.opentelemetry:shared-resourcemapping +com.google.cloud:google-cloud-monitorin com.google.code.findbugs:jsr305 com.google.code.gson:gson com.google.errorprone:error_prone_annotations @@ -308,6 +314,7 @@ io.grpc:grpc-googleapis io.grpc:grpc-grpclb io.grpc:grpc-inprocess io.grpc:grpc-netty-shaded +io.grpc:grpc-opentelemetry io.grpc:grpc-protobuf io.grpc:grpc-protobuf-lite io.grpc:grpc-rls @@ -354,6 +361,7 @@ io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi io.opentelemetry:opentelemetry-sdk-logs io.opentelemetry:opentelemetry-sdk-metrics io.opentelemetry:opentelemetry-sdk-trace +io.opentelemetry.contrib:opentelemetry-gcp-resources io.perfmark:perfmark-api io.projectreactor.netty:reactor-netty-core io.projectreactor.netty:reactor-netty-http @@ -394,6 +402,7 @@ org.apache.hadoop:hadoop-auth org.apache.hadoop:hadoop-client-api org.apache.hadoop:hadoop-common org.apache.hadoop:hadoop-hdfs-client +org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_21 org.apache.httpcomponents.client5:httpclient5 org.apache.httpcomponents.core5:httpcore5 org.apache.httpcomponents.core5:httpcore5-h2 @@ -3428,6 +3437,12 @@ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +--- +org.bouncycastle:bcprov-jdk15on + +Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sub license, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --- @@ -4785,7 +4800,10 @@ org.glassfish.hk2:osgi-resource-locator do so. If you do not wish to do so, delete this exception statement from your version. +--- +org.jline:jline +JLine is distributed under the BSD License, meaning that you are completely free to redistribute, modify, or sell it with almost no restrictions. --- org.latencyutils:LatencyUtils From a15f251efd1fd9186649b4ed52b9b5481e81cc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Sep 2024 18:04:35 +0200 Subject: [PATCH 02/36] Upgrade to guava 33.3.0-jre (#257) --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a67408cfc..7532f6c71 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -44,7 +44,7 @@ dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version = "4.0.7" } eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.4" } errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.29.2" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.42.0" } -guava = { module = "com.google.guava:guava", version = "33.2.1-jre" } +guava = { module = "com.google.guava:guava", version = "33.3.0-jre" } h2 = { module = "com.h2database:h2", version = "2.3.232" } hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "hadoop" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } From b3be8c4a44551fd17cf91a58c6fc5dec881e2d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Sep 2024 18:05:05 +0200 Subject: [PATCH 03/36] Upgrade to commons-codec 1.17.1 (#253) --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7532f6c71..d8d560d40 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -38,7 +38,7 @@ awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.26.31" } azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.25" } bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1.78.1" } caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.1.8" } -commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.0" } +commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.1" } commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" } dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version = "4.0.7" } eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.4" } From 2a3e437e00799f10adcf0d99427dcdf4a47b4cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Sep 2024 18:05:21 +0200 Subject: [PATCH 04/36] Upgrade to Azure SDK 1.2.27 (#251) --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d8d560d40..dae97ca6b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -35,7 +35,7 @@ swagger = "1.6.14" assertj-core = { module = "org.assertj:assertj-core", version = "3.26.3" } auth0-jwt = { module = "com.auth0:java-jwt", version = "4.2.1" } awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.26.31" } -azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.25" } +azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.27" } bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1.78.1" } caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.1.8" } commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.1" } From 9b0c78898b8c857f66346bf9a034e2849c1dffdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Sep 2024 18:05:49 +0200 Subject: [PATCH 05/36] Upgrade to java JWT 4.4.0 (#249) --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dae97ca6b..bafa88f68 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -33,7 +33,7 @@ swagger = "1.6.14" # (aka mention of the dependency removed). # assertj-core = { module = "org.assertj:assertj-core", version = "3.26.3" } -auth0-jwt = { module = "com.auth0:java-jwt", version = "4.2.1" } +auth0-jwt = { module = "com.auth0:java-jwt", version = "4.4.0" } awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.26.31" } azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.27" } bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1.78.1" } From 8f14363abc42ea6d95b67bc407f825b0af9bb4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Sep 2024 18:06:07 +0200 Subject: [PATCH 06/36] Upgrade to Dropwizard 4.0.8 (#248) This release includes a lot of dependency updates bringing fixes and security enhancements. You can take a look on the Release Notes for details: https://github.com/dropwizard/dropwizard/releases/tag/v4.0.8 --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bafa88f68..49a1106a7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -20,7 +20,7 @@ [versions] hadoop = "3.4.0" iceberg = "1.5.0" -dropwizard = "4.0.7" +dropwizard = "4.0.8" slf4j = "2.0.13" swagger = "1.6.14" @@ -40,7 +40,7 @@ bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.1.8" } commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.1" } commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" } -dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version = "4.0.7" } +dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version.ref = "dropwizard" } eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.4" } errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.29.2" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.42.0" } From d3cdb1cacfa93c64ca5a2ed1d6929b33c7f9d757 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 6 Sep 2024 11:32:34 +0200 Subject: [PATCH 07/36] License: add missing notices related to eclipselink (#271) --- LICENSE | 1959 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1937 insertions(+), 22 deletions(-) diff --git a/LICENSE b/LICENSE index 799d745cf..c094f2081 100644 --- a/LICENSE +++ b/LICENSE @@ -1348,6 +1348,230 @@ Linking this library statically or dynamically with other modules is making a co As a special exception, the copyright holders of this library give you permission to link this library with independent modules to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify this library, you may extend this exception to your version of the library, but you are not obligated to do so. If you do not wish to do so, delete this exception statement from your version. +--- +com.sun.xml.bind:jaxb-core + +Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +# Notices for Eclipse Implementation of JAXB + +This content is produced and maintained by the Eclipse Implementation of JAXB +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl + +## Trademarks + +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri + +## Third-party Content + +This project leverages the following third party content. + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Felix (1.2.0) + +* License: Apache License, 2.0 + +args4j (2.33) + +* License: MIT License + +dom4j (1.6.1) + +* License: Custom license based on Apache 1.1 + +file-management (3.0.0) + +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ + +JUnit (4.12) + +* License: Eclipse Public License + +JUnit (4.12) + +* License: Eclipse Public License + +maven-compat (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 + +maven-core (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 + +maven-plugin-annotations (3.5) + +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations + +maven-plugin-api (3.5.2) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-connector-basic (1.1.1) + +* License: Apache-2.0 + +maven-resolver-impl (1.1.1) + +* License: Apache-2.0 + +maven-resolver-spi (1.1.1) + +* License: Apache-2.0 + +maven-resolver-transport-file (1.1.1) + +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file + +maven-resolver-util (1.1.1) + +* License: Apache-2.0 + +maven-settings (3.5.2) + +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 + +OSGi Service Platform Core Companion Code (6.0) + +* License: Apache License, 2.0 + +plexus-archiver (3.5) + +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver + +plexus-io (3.0.0) + +* License: Apache-2.0 + +plexus-utils (3.1.0) + +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) + +relaxng-datatype (1.0) + +* License: New BSD license + +Sax (0.2) + +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 + +testng (6.14.2) + +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng + +wagon-http-lightweight (3.0.0) + +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 + +xz for java (1.8) + +* License: LicenseRef-Public-Domain + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. --- com.sun.xml.bind:jaxb-impl @@ -2110,6 +2334,255 @@ do so. If you do not wish to do so, delete this exception statement from your version. +--- +com.sun.xml.bind:jaxb-xjc + +Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +Copyright (c) 2004 Kohsuke Kawaguchi + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom +the Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + +# Notices for Eclipse Implementation of JAXB + +This content is produced and maintained by the Eclipse Implementation of JAXB +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl + +## Trademarks + +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri + +## Third-party Content + +This project leverages the following third party content. + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Felix (1.2.0) + +* License: Apache License, 2.0 + +args4j (2.33) + +* License: MIT License + +dom4j (1.6.1) + +* License: Custom license based on Apache 1.1 + +file-management (3.0.0) + +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ + +JUnit (4.12) + +* License: Eclipse Public License + +JUnit (4.12) + +* License: Eclipse Public License + +maven-compat (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 + +maven-core (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 + +maven-plugin-annotations (3.5) + +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations + +maven-plugin-api (3.5.2) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-connector-basic (1.1.1) + +* License: Apache-2.0 + +maven-resolver-impl (1.1.1) + +* License: Apache-2.0 + +maven-resolver-spi (1.1.1) + +* License: Apache-2.0 + +maven-resolver-transport-file (1.1.1) + +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file + +maven-resolver-util (1.1.1) + +* License: Apache-2.0 + +maven-settings (3.5.2) + +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 + +OSGi Service Platform Core Companion Code (6.0) + +* License: Apache License, 2.0 + +plexus-archiver (3.5) + +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver + +plexus-io (3.0.0) + +* License: Apache-2.0 + +plexus-utils (3.1.0) + +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) + +relaxng-datatype (1.0) + +* License: New BSD license + +Sax (0.2) + +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 + +testng (6.14.2) + +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng + +wagon-http-lightweight (3.0.0) + +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 + +xz for java (1.8) + +* License: LicenseRef-Public-Domain + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. --- @@ -2892,34 +3365,411 @@ permitted. --- -jakarta.xml.bind:jakarta.xml.bind-api +jakarta.persistence:jakarta.persistence-api -Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. +This program and the accompanying materials are made available under the +terms of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0, +or the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: +SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause + + +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. + + + +Eclipse Distribution License - v 1.0 +Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. +All rights reserved. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. +* Neither the name of the Eclipse Foundation, Inc. nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - Neither the name of the Eclipse Foundation, Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + +[//]: # " Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. " +[//]: # " " +[//]: # " This program and the accompanying materials are made available under the " +[//]: # " terms of the Eclipse Distribution License v. 1.0, which is available at " +[//]: # " http://www.eclipse.org/org/documents/edl-v10.php. " +[//]: # " " +[//]: # " SPDX-License-Identifier: BSD-3-Clause " + +# Notices for Jakarta Persistence + +This content is produced and maintained by the Jakarta Persistence +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jpa + +## Trademarks + + Jakarta Persistence is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0, or the Eclipse Distribution License v. 1.0 +which is available at http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jpa-api + +## Third-party Content + +This project leverages the following third party content. + +None + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + +--- +jakarta.xml.bind:jakarta.xml.bind-api + +Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @@ -3453,6 +4303,1071 @@ Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncy THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +--- +org.eclipse.angus:angus-activation + +Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +# Notices for Eclipse Angus + +This content is produced and maintained by the Eclipse Angus project. + +* Project home: https://projects.eclipse.org/projects/ee4j.angus + +## Trademarks + +Eclipse Angus is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v1.0 which is available at +https://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/angus-activation +* https://github.com/eclipse-ee4j/angus-mail + +## Third-party Content + +This project leverages the following third party content. + +None + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + + + +--- +org.eclipse.persistence:eclipselink + +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. + + + + + +Eclipse Distribution License - v 1.0 + +Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + Neither the name of the Eclipse Foundation, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +[//]: # " Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved. " +[//]: # " " +[//]: # " This program and the accompanying materials are made available under the " +[//]: # " terms of the Eclipse Public License v. 2.0 which is available at " +[//]: # " http://www.eclipse.org/legal/epl-2.0, " +[//]: # " or the Eclipse Distribution License v. 1.0 which is available at " +[//]: # " http://www.eclipse.org/org/documents/edl-v10.php. " +[//]: # " " +[//]: # " SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause " + +# Notices for EclipseLink + +This content is produced and maintained by the EclipseLink project. + +* Project home: https://projects.eclipse.org/projects/ee4j.eclipselink + +## Trademarks + + EclipseLink is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +https://www.eclipse.org/legal/epl-2.0, or the Eclipse Distribution License v1.0 +which is available at https://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/eclipselink +* https://github.com/eclipse-ee4j/eclipselink-workbench +* https://github.com/eclipse-ee4j/eclipselink-oracleddlparser +* https://github.com/eclipse-ee4j/eclipselink-examples +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/eclipselink.releng +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/eclipselink.runtime +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/eclipselink.utils.temp +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/examples +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/examples/mysports +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/examples/nosql +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/examples/performance +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/examples/temporal +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/incubator +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/javax.persistence +* https://git.eclipse.org/r/plugins/gitiles/eclipselink/oracleddlparser + +## Third-party Content + +This project leverages the following third party content. + +* Service Data Objects (SDO) (2.1) + +* License: OSOA SDO License + +Activation Framework (1.1) + +* License: Common Development and Distribution License + +ANTLR (3.0) + +* License: New BSD license + +ANTLR Runtime only (3.5.2) + +* License: New BSD License +* Project: http://www.antlr3.org/ +* Source: + http://repo1.maven.org/maven2/org/antlr/antlr-runtime/3.5.2/antlr-runtime-3.5.2-sources.jar + +ANTLR Runtime only: Version (3.2) + +* License: New BSD license + +Apache Ant (1.7.0) + +* License: Apache License, 2.0 + +Apache Ant (1.10.7) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Ant (1.10.7) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Geronimo Jaxws 2.1 Spec (1.0) + +* License: Apache License, 2.0 + +Apache Java Servlet API (2.4) + +* License: Apache License, 2.0 + +atinject (Package javax.inject) (1.0) + +* License: Apache License, 2.0 + +atinject (Package javax.inject) (1.0) + +* License: Apache License, 2.0 + +Bean Validation API (1.0) + +* License: Apache License, 2.0 +* Source: + http://anonsvn.jboss.org/repos/hibernate/beanvalidation/trunk/validation-api/ + +Bean Validation API (1.0.0) + +* License: Apache License, 2.0 +* Project: + http://repository.jboss.com/maven2/javax/validation/validation-api/1.0.0.GA/ +* Source: + http://repository.jboss.com/maven2/javax/validation/validation-api/1.0.0.GA/ + +Bean Validation API (1.1.0) + +* License: Apache License, 2.0 +* Project: http://beanvalidation.org/ + +Bean Validation API (2.0.1) + +* License: Apache-2.0 +* Project: http://beanvalidation.org/ +* Source: + https://github.com/beanvalidation/beanvalidation-api/releases/tag/2.0.1.Final + +bnd (0.0.351) + +* License: Apache License, 2.0 +* Project: http://sourceforge.net/projects/bnd/ +* Source: http://sourceforge.net/projects/bnd/ + +cdi-api (1.0) + +* License: Apache License, 2.0 + +cdi-api 2.0 (JSR 365: Contexts and Dependency Injection for Java (2.0) + +* License: Apache-2.0 +* Project: http://www.cdi-spec.org/ +* Source: + http://repo1.maven.org/maven2/javax/enterprise/cdi-api/2.0/cdi-api-2.0-sources.jar + +Classmate library (1.0.0) + +* License: Apache License, 2.0 +* Project: http://github.com/cowtowncoder/java-classmate +* Source: + https://github.com/cowtowncoder/java-classmate/archive/classmate-1.0.0.tar.gz + +commons-logging-1.1.1.jar (1.1.1) + +* License: Apache License, 2.0 +* Project: http://commons.apache.org/logging/ +* Source: + http://apache.siamwebhosting.com/commons/logging/binaries/commons-logging-1.1.1-bin.zip + +EJB (3.0) + +* License: Common Development and Distribution License +* Project: http://java.sun.com/products/ejb/ + +ejb-api (3.1.1) + +* License: Common Development and Distribution License + +Expression Language API (2.2) + +* License: Common Development and Distribution License, Apache 2.0 License +* Project: https://uel.java.net/ +* Source: + http://download.java.net/maven/2/javax/el/el-api/2.2/el-api-2.2-sources.jar + +Expression Language Implementation (2.2.2) + +* License: Common Development and Distribution License +* Project: https://uel.java.net/ +* Source: + http://search.maven.org/remotecontent?filepath=org/glassfish/web/javax.el/2.2.2/javax.el-2.2.2-sources.jar + +geronimo activation (1.1) + +* License: Apache License, 2.0 + +glassfish-corba-orb (4.2.0) + +* License: CDDL-1.1 OR GPL-2.0 With Classpath-exception-2.0 + +gmbal (4.0.0) + +* License: (CDDL-1.1 OR GPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0) AND + Apache-2.0 +* Project: https://javaee.github.io/gmbal/ +* Source: https://github.com/javaee/gmbal + +gmbal-api (3.2.0) + +* License: CDDL-1.1 or GPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 +* Project: https://javaee.github.io/gmbal/ +* Source: https://github.com/javaee/gmbal + +gmbal-pfl basic_4.0.1.b003 (4.0.1) + +* License: (CDDL-1.1 OR GPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0) AND + Apache-2.0 AND BSD-3-Clause + +hk2-api (2.3.0) + +* License: Common Development and Distribution License + +hk2-locator (2.3.0) + +* License: Common Development and Distribution License + +hk2-utils (2.3.0) + +* License: (CDDL-1.1 OR GPL-2.0 WITH Classpath-exception-2.0) + +Java API for JSON Processing JSR-353 (JSON-P) (0.0.99) + +* License: Common Development and Distribution License +* Project: http://json-processing-spec.java.net +* Source: http://java.net/projects/jsonp + +Java API for JSON Processing JSR-353 (JSON-P) (1.0) + +* License: Common Development and Distribution License +* Project: http://jsonp.java.net/ +* Source: + http://search.maven.org/remotecontent?filepath=javax/json/javax.json-api/1.0/javax.json-api-1.0-sources.jar + +Java API for JSON Processing RI JSR-353 (1.0) + +* License: Common Development and Distribution License +* Project: http://jsonp.java.net/ +* Source: + http://search.maven.org/remotecontent?filepath=org/glassfish/javax.json/1.0/javax.json-1.0-sources.jar + +Java Transaction (JTA) (1.1) + +* License: Common Development and Distribution License + +Java Transaction API (1.3) + +* License: (CDDL-1.1 OR GPL-2.0-only OR GPL-2.0-only WITH + classpath-exception-2.0) + +JavaCC (5.0) + +* License: New BSD License + +JavaMail (1.4) + +* License: Common Development and Distribution License + +Javax.annotation (1.2) + +* License: Common Development and Distribution License + +Javax.interceptor API (1.2) + +* License: Common Development and Distribution License 1.1 + +javax.json (1.0.4) + +* License: Common Development and Distribution License +* Project: https://jsonp.java.net +* Source: + http://central.maven.org/maven2/org/glassfish/javax.json/1.0.4/javax.json-1.0.4-sources.jar + +javax.json.bind-api (1.0) + +* License: Common Development and Distribution License +* Project: https://java.net/projects/jsonb-spec/pages/Home +* Source: https://java.net/projects/jsonb-spec/sources/git/show + +javax.ws.rs (2.0.1) + +* License: Common Development and Distribution License, + 1 file partial ASL + +JAX-RS (JSR311) API (1.1.1) + +* License: Common Development and Distribution License +* Project: https://jsr311.dev.java.net/ +* Source: + http://download.java.net/maven/2/javax/ws/rs/jsr311-api/1.1/jsr311-api-1.1-sources.jar + +JAXB (2.1.9) + +* License: Common Development and Distribution License + +JAXB (2.1.9) + +* License: Common Development and Distribution License + +JAXB (2.1.12) + +* License: BSD, Apache 2.0, CDDL, Public Domain, MIT License, Apache 1.1 + +JAXB 2.0 Reference Implementation (jaxb-impl.jar) (2.0.5) + +* License: Common Development and Distribution License + +JAXB IMPL (2.1.12) + +* License: Common Development and Distribution License +* Project: https://jaxb.dev.java.net/2.1.12/ +* Source: https://jaxb.dev.java.net/2.1.12/JAXB2_20090708.jar + +JAXB 2.0 XJC (2.0.5) + +* License: Common Development and Distribution License + +JAXB 2.2 API (Binary only) (2.2) + +* License: Common Development and Distribution License +* Project: https://jaxb.dev.java.net/2.2/ +* Source: https://jaxb.dev.java.net/2.2/JAXB2_20091104.jar + +JAXB 2.2 Impl (Binary only) (2.2) + +* License: Common Development and Distribution License + +JAXB 2.2 XJC (Binary) (2.2) + +* License: BSD, CDDL, Public Domain +* Project: https://jaxb.dev.java.net/2.2/ +* Source: https://jaxb.dev.java.net/2.2/JAXB2_20091104.jar + +JAXB API (2.2.12) + +* License: Common Development and Distribution License +* Project: https://jaxb.java.net/ +* Source: + https://maven.java.net/service/local/artifact/maven/redirect?r=metro-388&g=javax.xml.bind&a=jaxb-api&v=2.2.12-b140109.1041&e=jar&c=sources + +JAXB API (2.1.12) + +* License: Common Development and Distribution License +* Project: https://jaxb.dev.java.net/2.1.12/ +* Source: https://jaxb.dev.java.net/2.1.12/JAXB2_20090708.jar + +JAXB CORE (2.2.11) + +* License: Common Development and Distribution License + +JAXB CORE (2.2.11) + +* License: Common Development and Distribution License +* Project: https://jaxb.java.net/ +* Source: + https://maven.java.net/service/local/artifact/maven/redirect?r=metro-535&g=org.glassfish.jaxb&a=jaxb-core&v=2.2.11-M1&e=jar&c=sources + +JAXB IMPL (2.2.11) + +* License: Common Development and Distribution License + +JAXB XJC (2.2.11) + +* License: Common Development and Distribution License, MIT, BSD, Public + Domain, Apache License, 2.0 +* Project: https://jaxb.java.net/ +* Source: + https://maven.java.net/service/local/artifact/maven/redirect?r=metro-535&g=org.glassfish.jaxb&a=jaxb-xjc&v=2.2.11-M1&e=jar&c=sources + +JAXB-API 2.0 (2.0) + +* License: Commond Development and Distribution License + +jaxrpc.jar (1.1) + +* License: Common Development and Distribution License + +Jaxws-api-2.0.jar (2.0) + +* License: Common Development and Distribution License + +JCA 1.6 (1.6) + +* License: Common Development and Distribution License + +JCA Connector (1.5) + +* License: Common Development and Distribution License + +Jersey Common (2.0) + +* License: Common Development and Distribution License +* Project: http://jersey.java.net/ +* Source: https://github.com/jersey/jersey + +Jersey Core (1.8.0) + +* License: CDDL, Apache 2.0 (four files) + +Jersey Guava Repackaged (2.14) + +* License: Apache License, 2.0 + +Jersey Server (2.0) + +* License: Common Development and Distribution License +* Project: http://jersey.java.net/ +* Source: https://github.com/jersey/jersey + +jersey-client (2.14) + +* License: Common Development and Distribution License + +jersey-common (2.14) + +* License: Common Development and Distribution License + +jgroups (4.1.8) + +* License: Apache-2.0 AND CC-BY-2.5 AND LicenseRef-Public-Domain + +JMS (1.1) + +* License: Common Development and Distribution License + +JPA (2.0) + +* License: Negotiated agreement between Sun and Eclipse (supercedes spec terms) +* Project: http://jcp.org/en/jsr/detail?id=317 + +JPA (Javax Persistence Jar) (1.0) + +* License: Common Development and Distribution License + +Logback Classic (1.0.7) + +* License: Eclipse Public License + +Logback Core (1.0.7) + +* License: Eclipse Public License + +logback-classic (1.3.0) + +* License: EPL-1.0 OR LGPL-2.1 + +management-api (3.2.1) + +* License: (CDDL-1.1 OR GPL-2.0-only OR GPL-2.0-only WITH + Classpath-exception-2.0) +* Project: https://javaee.github.io/gmbal-commons/ +* Source: https://github.com/javaee/gmbal-commons + +MongoDB Java Driver (2.7.3) + +* License: Apache License, 2.0, Eclipse Public License (One File) + +MongoDB Java Driver (2.10.1) + +* License: Apache License, 2.0 + +mongodb java driver (3.2.0) + +* License: Apache License, 2.0, Creative Commons Attribution License 2.5, + Public Domain +* Project: https://docs.mongodb.org/ecosystem/drivers/java/ +* Source: + http://central.maven.org/maven2/org/mongodb/mongo-java-driver/3.2.0/mongo-java-driver-3.2.0-sources.jar + +mongodb java driver (3.11.2) + +* License: Apache-2.0 AND MIT AND AND CC-BY-2.5 AND CC0-1.0 + +opencsv (1.8) + +* License: Apache License, 2.0 +* Project: http://opencsv.sourceforge.net/ +* Source: + http://downloads.sourceforge.net/opencsv/opencsv-1.8-src-with-libs.tar.gz?modtime=1185864370&big_mirror=0 + +oracle-nosql-client (18.3.10) + +* License: Apache-2.0 + +org.apache.ant (1.6.5) + +* License: Apache License, 2.0 +* Project: http://ant.apache.org/ + +org.apache.felix.framework (6.0.3) + +* License: Pending + +org.osgi.core (1.2.0) + +* License: Apache-2.0 + +org.osgi.core (6.0.0) + +* License: Apache-2.0 + +OSGi Enterprise Specification APIs (4.2) + +* License: Apache License, 2.0 + +osgi.compendium (4.1.0) + +* License: Apache License, 2.0 + +osgi.core (4.1.0) + +* License: Apache License, 2.0 + +pax-exam (n/a) + +* License: Pending + +pax-exam-container-forked (4.13.1) + +* License: Pending + +pax-exam-junit4 (4.13.1) + +* License: Pending + +pax-exam-link-mvn (4.13.1) + +* License: Pending + +saaj-api-1.3.jar (1.3) + +* License: Common Development and Distribution License + +sdo Version: SDO (2.1.1) + +* License: SDO License (Custom) +* Project: http://jcp.org/aboutJava/communityprocess/pr/jsr235/index.html + +SLF4J API (1.7.2) + +* License: MIT License + +slf4j-api (1.7.30) + +* License: MIT + +slf4j-api (2.0.0) + +* License: MIT + +spring-agent (2.5.3) + +* License: Apache License, 2.0 +* Project: http://www.springframework.org +* Source: + http://downloads.sourceforge.net/springframework/spring-framework-2.5.3-with-dependencies.zip?modtime=1204283106&big_mirror=0 + +spring-aop (5.2.0) + +* License: Apache-2.0 + +spring-beans (5.2.0) + +* License: Apache-2.0 + +spring-context (5.2.0) + +* License: Apache-2.0 + +spring-core (5.2.0) + +* License: Apache-2.0 AND BSD-3-Clause + +spring-expression (5.2.0) + +* License: Apache-2.0 + +spring-instrument (5.2.0) + +* License: Pending + +spring-jcl (5.2.0) + +* License: Apache-2.0 + +spring-jdbc (5.2.0) + +* License: Pending + +spring-orm (5.2.0) + +* License: Pending + +spring-test (5.2.0) + +* License: Apache-2.0 AND BSD-3-Clause AND LicenseRef-Public-Domain + +spring-tx (5.2.0) + +* License: Pending + +StAX-API.Jar (1.0.1) + +* License: Apache License, 2.0 + +wsdl4j (1.6.2) + +* License: Common Public License 1.0 + +Xerces (2.9.0) + +* License: Apache License, 2.0 + +Xerces (2.12.0) + +* License: Apache-2.0 AND W3C-19980720 +* Project: http://xerces.apache.org/xerces2-j/ +* Source: https://svn.apache.org/repos/asf/xerces/java/tags/Xerces-J_2_12_0/ + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + + + +License for the Service Data Objects JavaDoc and Interface Definition +files. + +The Service Data Objects JavaDoc and Interface Definition files are +being provided by the copyright holders under the following license. By +using and/or copying this work, you agree that you have read, +understood and will comply with the following terms and conditions: + +Permission to copy, display, make derivative works of and distribute +the Service Data Objects JavaDoc and Interface Definition files (the +"Artifacts") in any medium without fee or royalty is hereby granted, +provided that you include the following on ALL copies of the Artifacts, +or portions thereof, that you make: + +1. A link or URL to the Artifacts at this location: +http://www.jcp.org/en/jsr/detail?id=235 + +2. The full text of this copyright notice as shown in the Artifacts. + + + +THE ARTIFACTS ARE PROVIDED "AS IS" AND THE AUTHORS MAKE NO +REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THE +ARTIFACTS AND THE IMPLEMENTATION OF THEIR CONTENTS, +INCLUDING, BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT OR TITLE. + +THE AUTHORS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATING TO ANY +USE OR DISTRIBUTION OF THE ARTIFACTS. + +The name and trademarks of the Authors may NOT be used in any manner, +including advertising or publicity pertaining to the Service Data +Objects Specification or its contents without specific, written prior +permission. Title to copyright in the Service Data Objects +Specification will at all times remain with the Authors. + +No other rights are granted by implication, estoppel or otherwise. + +Revision level 1.2, last updated on 2009/01/13 +Changed the URL of the Artifacts. +Revision level 1.1, last updated on 2007/11/19 + --- org.glassfish.hk2.external:aopalliance-repackaged From 96284a94bb734ab980b845cf01e49bbd314a37bf Mon Sep 17 00:00:00 2001 From: "Hung-An (Alvin) Chen" Date: Fri, 6 Sep 2024 09:24:01 -0500 Subject: [PATCH 08/36] Notification API reject out of order notifications (#232) --- .../polaris/core/entity/TableLikeEntity.java | 16 +++++ .../service/catalog/BasePolarisCatalog.java | 15 +++- .../catalog/BasePolarisCatalogTest.java | 68 ++++++++++++++++++- spec/rest-catalog-open-api.yaml | 32 ++++++++- 4 files changed, 128 insertions(+), 3 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java index b5ce9bb4f..968598b93 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.entity; import com.fasterxml.jackson.annotation.JsonIgnore; +import java.util.Optional; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; @@ -31,6 +32,9 @@ public class TableLikeEntity extends PolarisEntity { public static final String USER_SPECIFIED_WRITE_DATA_LOCATION_KEY = "write.data.path"; public static final String USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY = "write.metadata.path"; + public static final String LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY = + "last-notification-timestamp"; + public TableLikeEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); } @@ -63,6 +67,13 @@ public String getMetadataLocation() { return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY); } + @JsonIgnore + public Optional getLastAdmittedNotificationTimestamp() { + return Optional.ofNullable( + getInternalPropertiesAsMap().get(LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY)) + .map(Long::parseLong); + } + @JsonIgnore public String getBaseLocation() { return getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION); @@ -109,5 +120,10 @@ public Builder setMetadataLocation(String location) { internalProperties.put(METADATA_LOCATION_KEY, location); return this; } + + public Builder setLastNotificationTimestamp(long timestamp) { + internalProperties.put(LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY, String.valueOf(timestamp)); + return this; + } } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 2538caf50..848e10695 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1874,10 +1874,23 @@ private boolean sendNotificationForTableLike( .getMetaStoreManager() .generateNewEntityId(getCurrentPolarisContext()) .getId()) + .setLastNotificationTimestamp(request.getPayload().getTimestamp()) .build(); } else { + // If the notification timestamp is out-of-order, we should not update the table + if (entity.getLastAdmittedNotificationTimestamp().isPresent() + && request.getPayload().getTimestamp() + <= entity.getLastAdmittedNotificationTimestamp().get()) { + throw new AlreadyExistsException( + "A notification with a newer timestamp has been processed for table %s", + tableIdentifier); + } existingLocation = entity.getMetadataLocation(); - entity = new TableLikeEntity.Builder(entity).setMetadataLocation(newLocation).build(); + entity = + new TableLikeEntity.Builder(entity) + .setMetadataLocation(newLocation) + .setLastNotificationTimestamp(request.getPayload().getTimestamp()) + .build(); } // first validate we can read the metadata file validateLocationForTableLike(tableIdentifier, newLocation); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index c574e5f42..1a69c7bac 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -47,6 +47,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.BadRequestException; import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; @@ -623,7 +624,7 @@ public void testUpdateNotificationCreateTableInExternalLocation() { update.setMetadataLocation(maliciousMetadataFile); update.setTableName(table.name()); update.setTableUuid(UUID.randomUUID().toString()); - update.setTimestamp(230950845L); + update.setTimestamp(230950849L); updateRequest.setPayload(update); Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, updateRequest)) @@ -913,6 +914,71 @@ public void testUpdateNotificationWhenTableExistsInDisallowedLocation() { .hasMessageContaining("Invalid location"); } + @Test + public void testUpdateNotificationRejectOutOfOrderTimestamp() { + Assumptions.assumeTrue( + requiresNamespaceCreate(), + "Only applicable if namespaces must be created before adding children"); + Assumptions.assumeTrue( + supportsNestedNamespaces(), "Only applicable if nested namespaces are supported"); + Assumptions.assumeTrue( + supportsNotifications(), "Only applicable if notifications are supported"); + + final String tableLocation = "s3://externally-owned-bucket/table/"; + final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; + BasePolarisCatalog catalog = catalog(); + + Namespace namespace = Namespace.of("parent", "child1"); + TableIdentifier table = TableIdentifier.of(namespace, "table"); + + long timestamp = 230950845L; + NotificationRequest request = new NotificationRequest(); + request.setNotificationType(NotificationType.CREATE); + TableUpdateNotification update = new TableUpdateNotification(); + update.setMetadataLocation(tableMetadataLocation); + update.setTableName(table.name()); + update.setTableUuid(UUID.randomUUID().toString()); + update.setTimestamp(timestamp); + request.setPayload(update); + + InMemoryFileIO fileIO = (InMemoryFileIO) catalog.getIo(); + + fileIO.addFile( + tableMetadataLocation, + TableMetadataParser.toJson(createSampleTableMetadata(tableLocation)).getBytes(UTF_8)); + + catalog.sendNotification(table, request); + + // Send a notification with a timestamp same as that of the previous notification, should fail + NotificationRequest request2 = new NotificationRequest(); + request2.setNotificationType(NotificationType.UPDATE); + TableUpdateNotification update2 = new TableUpdateNotification(); + update2.setMetadataLocation(tableLocation + "metadata/v2.metadata.json"); + update2.setTableName(table.name()); + update2.setTableUuid(UUID.randomUUID().toString()); + update2.setTimestamp(timestamp); + request2.setPayload(update2); + + Assertions.assertThatThrownBy(() -> catalog.sendNotification(table, request2)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining( + "A notification with a newer timestamp has been processed for table parent.child1.table"); + + // Verify that DROP notification won't be rejected due to timestamp + NotificationRequest request3 = new NotificationRequest(); + request3.setNotificationType(NotificationType.DROP); + TableUpdateNotification update3 = new TableUpdateNotification(); + update3.setMetadataLocation(tableLocation + "metadata/v2.metadata.json"); + update3.setTableName(table.name()); + update3.setTableUuid(UUID.randomUUID().toString()); + update3.setTimestamp(timestamp); + request3.setPayload(update3); + + Assertions.assertThat(catalog.sendNotification(table, request3)) + .as("Drop notification should not fail despite timestamp being outdated") + .isTrue(); + } + @Test public void testUpdateNotificationWhenTableExistsFileSpecifiesDisallowedLocation() { Assumptions.assumeTrue( diff --git a/spec/rest-catalog-open-api.yaml b/spec/rest-catalog-open-api.yaml index 9c7e61e75..6bea02acb 100644 --- a/spec/rest-catalog-open-api.yaml +++ b/spec/rest-catalog-open-api.yaml @@ -976,7 +976,14 @@ paths: summary: Sends a notification to the table operationId: sendNotification requestBody: - description: The request containing the notification to be sent + description: + The request containing the notification to be sent. + + For each table, Polaris will reject any notification where the timestamp in the request body + is older than or equal to the most recent time Polaris has already processed for the table. + The responsibility of ensuring the correct order of timestamps for a sequence of notifications + lies with the caller of the API. This includes managing potential clock skew or inconsistencies + when notifications are sent from multiple sources. content: application/json: schema: @@ -1001,6 +1008,16 @@ paths: examples: TableToLoadDoesNotExist: $ref: '#/components/examples/NoSuchTableError' + 409: + description: + Conflict - The timestamp of the received notification is older than or equal to the + most recent timestamp Polaris has already processed for the specified table. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: + $ref: '#/components/examples/OutOfOrderNotificationError' 419: $ref: '#/components/responses/AuthenticationTimeoutResponse' 503: @@ -3213,6 +3230,7 @@ components: NotificationRequest: required: - notification-type + - payload properties: notification-type: $ref: '#/components/schemas/NotificationType' @@ -4130,6 +4148,18 @@ components: "updates": {"owner": "Raoul"} } + OutOfOrderNotificationError: + summary: + The timestamp of the received notification is older than or equal to the most recent timestamp + Polaris has already processed for the specified table. + value: { + "error": { + "message": "A notification with a newer timestamp has been admitted for table", + "type": "AlreadyExistsException", + "code": 409 + } + } + securitySchemes: OAuth2: type: oauth2 From 8e04a352efb1168e3f7df8a261c74fcc8554334c Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Fri, 6 Sep 2024 23:25:46 +0900 Subject: [PATCH 09/36] Remove unused method (#221) --- .../java/org/apache/polaris/service/auth/TokenUtils.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java b/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java index c04e42704..366f0ed88 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/auth/TokenUtils.java @@ -31,12 +31,6 @@ public class TokenUtils { - /** Get token against default realm */ - public static String getTokenFromSecrets( - Client client, int port, String clientId, String clientSecret) { - return getTokenFromSecrets(client, port, clientId, clientSecret, null); - } - /** Get token against specified realm */ public static String getTokenFromSecrets( Client client, int port, String clientId, String clientSecret, String realm) { From ec6ffd0a274cb718a9c54dbf881d5d7342a0ce38 Mon Sep 17 00:00:00 2001 From: Dmitri Bourlatchkov Date: Mon, 9 Sep 2024 08:04:46 -0400 Subject: [PATCH 10/36] Add python aux files to .gitignore (#246) After running `./polaris ...`: ``` $ git st On branch main Your branch is up to date with 'origin/main'. Untracked files: (use "git add ..." to include in what will be committed) poetry.lock polaris-venv/ pyproject.toml ``` --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 8dfa578d7..f9b35731c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,11 @@ regtests/derby.log regtests/metastore_db regtests/output/ +# Python stuff +/poetry.lock +/polaris-venv/ +/pyproject.toml + # Notebooks notebooks/.ipynb_checkpoints/ From 17dd740dec600af55ce29e2f81c61192bb4bc21c Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Mon, 9 Sep 2024 05:12:37 -0700 Subject: [PATCH 11/36] Remove /metadata and /data subdirectory locations from generated subscoped policy (#244) --------- Co-authored-by: Michael Collado --- .../service/catalog/BasePolarisCatalog.java | 12 ++-------- .../src/test_spark_sql_s3_with_privileges.py | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 848e10695..164e13406 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.catalog; -import static org.apache.polaris.core.storage.StorageUtil.concatFilePrefixes; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Objects; @@ -364,10 +362,8 @@ private String defaultNamespaceLocation(Namespace namespace) { } private Set getLocationsAllowedToBeAccessed(TableMetadata tableMetadata) { - String basicLocation = tableMetadata.location(); Set locations = new HashSet<>(); - locations.add(concatFilePrefixes(basicLocation, "data/", "/")); - locations.add(concatFilePrefixes(basicLocation, "metadata/", "/")); + locations.add(tableMetadata.location()); if (tableMetadata .properties() .containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)) { @@ -386,11 +382,7 @@ private Set getLocationsAllowedToBeAccessed(TableMetadata tableMetadata) } private Set getLocationsAllowedToBeAccessed(ViewMetadata viewMetadata) { - String basicLocation = viewMetadata.location(); - Set locations = new HashSet<>(); - // a view won't have a "data" location, so only allowed to access "metadata" - locations.add(concatFilePrefixes(basicLocation, "metadata/", "/")); - return locations; + return Set.of(viewMetadata.location()); } @Override diff --git a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py index f58e91127..a6c216a63 100644 --- a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py +++ b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py @@ -805,6 +805,12 @@ def test_spark_credentials_s3_direct_with_write(root_client, snowflake_catalog, aws_secret_access_key=response.config['s3.secret-access-key'], aws_session_token=response.config['s3.session-token']) + objects = s3.list_objects(Bucket=test_bucket, Delimiter='/', + Prefix='polaris_test/snowflake_catalog/db1/schema/iceberg_table/') + assert objects is not None + assert 'CommonPrefixes' in objects + assert len(objects['CommonPrefixes']) > 0 + objects = s3.list_objects(Bucket=test_bucket, Delimiter='/', Prefix='polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/') assert objects is not None @@ -1051,7 +1057,7 @@ def test_spark_credentials_s3_scoped_to_metadata_data_locations(root_client, sno objects = client.list_objects(Bucket=test_bucket, Delimiter='/', Prefix=f'{prefix}/metadata/') assert objects is not None - assert 'Contents' in objects , f'list medata files failed in prefix: {prefix}/metadata/' + assert 'Contents' in objects , f'list metadata files failed in prefix: {prefix}/metadata/' objects = client.list_objects(Bucket=test_bucket, Delimiter='/', Prefix=f'{prefix}/data/') @@ -1059,15 +1065,11 @@ def test_spark_credentials_s3_scoped_to_metadata_data_locations(root_client, sno # no insert executed, so should not have any data files assert 'Contents' not in objects , f'No contents should be in prefix: {prefix}/data/' - # list files fail in the same table's other directory. The access policy should restrict this - # even metadata and data, it needs an ending `/` - for invalidPrefix in [f'{prefix}/other_directory/', f'{prefix}/metadata', f'{prefix}/data']: - try: - client.list_objects(Bucket=test_bucket, Delimiter='/', - Prefix=invalidPrefix) - pytest.fail(f'Expected exception listing files outside of allowed table directories, but succeeds on location: {invalidPrefix}') - except botocore.exceptions.ClientError as error: - assert error.response['Error']['Code'] == 'AccessDenied', 'Expected exception AccessDenied, but got: ' + error.response['Error']['Code'] + ' on location: ' + invalidPrefix + objects = client.list_objects(Bucket=test_bucket, Delimiter='/', + Prefix=f'{prefix}/') + assert objects is not None + assert 'CommonPrefixes' in objects , f'list prefixes failed in prefix: {prefix}/' + assert len(objects['CommonPrefixes']) > 0 with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', catalog_name=snowflake_catalog.name, From 7d1b8f21eba40009f9d42ccb7be580faf26bdeaa Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 14:52:57 -0700 Subject: [PATCH 12/36] Add a configuration to optionally disable drop with purge (#270) --- .../polaris/core/PolarisConfiguration.java | 25 ++++++++ .../service/admin/PolarisAdminService.java | 3 +- .../service/catalog/BasePolarisCatalog.java | 27 +++++++- .../catalog/BasePolarisCatalogTest.java | 61 +++++++++++++++++++ 4 files changed, 112 insertions(+), 4 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index 31af621d8..ee101997d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -150,4 +150,29 @@ public static Builder builder() { "If set to true, allows tables to have external locations outside the default structure.") .defaultValue(false) .build(); + + public static final PolarisConfiguration CLEANUP_ON_NAMESPACE_DROP = + PolarisConfiguration.builder() + .key("CLEANUP_ON_NAMESPACE_DROP") + .catalogConfig("cleanup.on.namespace.drop") + .description("If set to true, clean up data when a namespace is dropped") + .defaultValue(false) + .build(); + + public static final PolarisConfiguration CLEANUP_ON_CATALOG_DROP = + PolarisConfiguration.builder() + .key("CLEANUP_ON_CATALOG_DROP") + .catalogConfig("cleanup.on.catalog.drop") + .description("If set to true, clean up data when a catalog is dropped") + .defaultValue(false) + .build(); + + public static final PolarisConfiguration DROP_WITH_PURGE_ENABLED = + PolarisConfiguration.builder() + .key("DROP_WITH_PURGE_ENABLED") + .catalogConfig("drop-with-purge.enabled") + .description( + "If set to true, allows tables to be dropped with the purge parameter set to true.") + .defaultValue(true) + .build(); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index ebe7d4df6..023c96f13 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -97,7 +97,6 @@ */ public class PolarisAdminService { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisAdminService.class); - public static final String CLEANUP_ON_CATALOG_DROP = "CLEANUP_ON_CATALOG_DROP"; private final CallContext callContext; private final PolarisEntityManager entityManager; @@ -578,7 +577,7 @@ public void deleteCatalog(String name) { boolean cleanup = polarisCallContext .getConfigurationStore() - .getConfiguration(polarisCallContext, CLEANUP_ON_CATALOG_DROP, false); + .getConfiguration(polarisCallContext, PolarisConfiguration.CLEANUP_ON_CATALOG_DROP); PolarisMetaStoreManager.DropEntityResult dropEntityResult = entityManager .getMetaStoreManager() diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 164e13406..d27cc82c4 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -155,7 +155,6 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog && !(ex instanceof UnprocessableEntityException) && isStorageProviderRetryableException(ex); }; - public static final String CLEANUP_ON_NAMESPACE_DROP = "CLEANUP_ON_NAMESPACE_DROP"; private final PolarisEntityManager entityManager; private final CallContext callContext; @@ -620,7 +619,8 @@ public boolean dropNamespace(Namespace namespace) throws NamespaceNotEmptyExcept Map.of(), polarisCallContext .getConfigurationStore() - .getConfiguration(polarisCallContext, CLEANUP_ON_NAMESPACE_DROP, false)); + .getConfiguration( + polarisCallContext, PolarisConfiguration.CLEANUP_ON_NAMESPACE_DROP)); if (!dropEntityResult.isSuccess() && dropEntityResult.failedBecauseNotEmpty()) { throw new NamespaceNotEmptyException("Namespace %s is not empty", namespace); @@ -1803,6 +1803,7 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { } } + @SuppressWarnings("FormatStringAnnotation") private @NotNull PolarisMetaStoreManager.DropEntityResult dropTableLike( PolarisEntitySubType subType, TableIdentifier identifier, @@ -1818,6 +1819,28 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { List catalogPath = resolvedEntities.getRawParentPath(); PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); + + // Check that purge is enabled, if it is set: + if (catalogPath != null && !catalogPath.isEmpty() && purge) { + boolean dropWithPurgeEnabled = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getPolarisCallContext(), + catalogEntity, + PolarisConfiguration.DROP_WITH_PURGE_ENABLED); + if (!dropWithPurgeEnabled) { + throw new ForbiddenException( + String.format( + "Unable to purge entity: %s. To enable this feature, set the Polaris configuration %s " + + "or the catalog configuration %s", + identifier.name(), + PolarisConfiguration.DROP_WITH_PURGE_ENABLED.key, + PolarisConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig())); + } + } + return entityManager .getMetaStoreManager() .dropEntityIfExists( diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 1a69c7bac..c16b93cf9 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -1212,6 +1212,67 @@ public void testDropTableWithPurge() { Assertions.assertThat(fileIO).isNotNull().isInstanceOf(InMemoryFileIO.class); } + @Test + public void testDropTableWithPurgeDisabled() { + // Create a catalog with purge disabled: + String noPurgeCatalogName = CATALOG_NAME + "_no_purge"; + String storageLocation = "s3://testDropTableWithPurgeDisabled/data"; + AwsStorageConfigInfo noPurgeStorageConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::012345678901:role/jdoe") + .setExternalId("externalId") + .setUserArn("aws::a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .build(); + adminService.createCatalog( + new CatalogEntity.Builder() + .setName(noPurgeCatalogName) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty(PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .addProperty(PolarisConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") + .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) + .build()); + RealmContext realmContext = () -> "realm"; + CallContext callContext = CallContext.of(realmContext, polarisContext); + PolarisPassthroughResolutionView passthroughView = + new PolarisPassthroughResolutionView( + callContext, entityManager, authenticatedRoot, noPurgeCatalogName); + BasePolarisCatalog noPurgeCatalog = + new BasePolarisCatalog( + entityManager, + callContext, + passthroughView, + authenticatedRoot, + Mockito.mock(), + new DefaultFileIOFactory()); + noPurgeCatalog.initialize( + noPurgeCatalogName, + ImmutableMap.of( + CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); + + if (this.requiresNamespaceCreate()) { + ((SupportsNamespaces) noPurgeCatalog).createNamespace(NS); + } + + Assertions.assertThatPredicate(noPurgeCatalog::tableExists) + .as("Table should not exist before create") + .rejects(TABLE); + + Table table = noPurgeCatalog.buildTable(TABLE, SCHEMA).create(); + Assertions.assertThatPredicate(noPurgeCatalog::tableExists) + .as("Table should exist after create") + .accepts(TABLE); + Assertions.assertThat(table).isInstanceOf(BaseTable.class); + + // Attempt to drop the table: + Assertions.assertThatThrownBy(() -> noPurgeCatalog.dropTable(TABLE, true)) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining(PolarisConfiguration.DROP_WITH_PURGE_ENABLED.key); + } + private TableMetadata createSampleTableMetadata(String tableLocation) { Schema schema = new Schema( From bfe3f2f87068583ba71ec12a0028301adb477c1a Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 9 Sep 2024 15:02:12 -0700 Subject: [PATCH 13/36] Support buckets containing underscore (#263) --- .../polaris/core/storage/StorageUtil.java | 22 ++++++ .../aws/AwsCredentialsStorageIntegration.java | 4 +- .../gcp/GcpCredentialsStorageIntegration.java | 3 +- .../polaris/core/storage/StorageUtilTest.java | 69 +++++++++++++++++++ ...PolarisStorageIntegrationProviderImpl.java | 2 +- 5 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/storage/StorageUtilTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageUtil.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageUtil.java index d082ae1b0..13c0f17e0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageUtil.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageUtil.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.core.storage; +import java.net.URI; import org.jetbrains.annotations.NotNull; public class StorageUtil { @@ -40,4 +41,25 @@ public class StorageUtil { return leftPath + rightPath; } } + + /** + * Given a path, extract the bucket (authority). + * + * @param path A path to parse + * @return The bucket/authority of the path + */ + public static @NotNull String getBucket(String path) { + URI uri = URI.create(path); + return getBucket(uri); + } + + /** + * Given a URI, extract the bucket (authority). + * + * @param uri A path to parse + * @return The bucket/authority of the URI + */ + public static @NotNull String getBucket(URI uri) { + return uri.getAuthority(); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index c2b865f8e..864f72fa4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -112,7 +112,7 @@ private IamPolicy policyString( if (allowList) { bucketListStatmentBuilder .computeIfAbsent( - arnPrefix + uri.getHost(), + arnPrefix + StorageUtil.getBucket(uri), (String key) -> IamStatement.builder() .effect(IamEffect.ALLOW) @@ -164,7 +164,7 @@ private String getArnPrefixFor(String roleArn) { } private static @NotNull String parseS3Path(URI uri) { - String bucket = uri.getHost(); + String bucket = StorageUtil.getBucket(uri); String path = trimLeadingSlash(uri.getPath()); return String.join( "/", Stream.of(bucket, path).filter(Objects::nonNull).toArray(String[]::new)); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index a569d32a3..305690923 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -41,6 +41,7 @@ import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.StorageUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -139,7 +140,7 @@ public static CredentialAccessBoundary generateAccessBoundaryRules( .forEach( location -> { URI uri = URI.create(location); - String bucket = uri.getHost(); + String bucket = StorageUtil.getBucket(uri); readBuckets.add(bucket); String path = uri.getPath().substring(1); List resourceExpressions = diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageUtilTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageUtilTest.java new file mode 100644 index 000000000..8e842d243 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageUtilTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class StorageUtilTest { + + @Test + public void testEmptyString() { + Assertions.assertThat(StorageUtil.getBucket("")).isNull(); + } + + @ParameterizedTest + @ValueSource(strings = {"s3", "gcs", "abfs", "file"}) + public void testAbsolutePaths(String scheme) { + Assertions.assertThat(StorageUtil.getBucket(scheme + "://bucket/path/file.txt")) + .isEqualTo("bucket"); + Assertions.assertThat(StorageUtil.getBucket(scheme + "://bucket:with:colon/path/file.txt")) + .isEqualTo("bucket:with:colon"); + Assertions.assertThat(StorageUtil.getBucket(scheme + "://bucket_with_underscore/path/file.txt")) + .isEqualTo("bucket_with_underscore"); + Assertions.assertThat(StorageUtil.getBucket(scheme + "://bucket_with_ユニコード/path/file.txt")) + .isEqualTo("bucket_with_ユニコード"); + } + + @Test + public void testRelativePaths() { + Assertions.assertThat(StorageUtil.getBucket("bucket/path/file.txt")).isNull(); + Assertions.assertThat(StorageUtil.getBucket("path/file.txt")).isNull(); + } + + @Test + public void testAbsolutePathWithoutScheme() { + Assertions.assertThat(StorageUtil.getBucket("/bucket/path/file.txt")).isNull(); + } + + @Test + public void testInvalidURI() { + Assertions.assertThatThrownBy( + () -> StorageUtil.getBucket("s3://bucket with space/path/file.txt")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testAuthorityWithPort() { + Assertions.assertThat(StorageUtil.getBucket("s3://bucket:8080/path/file.txt")) + .isEqualTo("bucket:8080"); + } +} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 518ed2b97..3fb034130 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -73,7 +73,7 @@ PolarisStorageIntegration getStorageIntegrationForConfig( HttpTransportFactory.class, NetHttpTransport::new)); } catch (IOException e) { throw new RuntimeException( - "Error initializing default google credentials" + e.getMessage()); + "Error initializing default google credentials. " + e.getMessage()); } break; case AZURE: From aac0e526a3165330ae05a1ba9e6b172c75181a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 10 Sep 2024 11:43:01 +0200 Subject: [PATCH 14/36] Upgrade to Gradle 8.10.1 (#277) --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 1fa499edd..fdd85d1d9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -20,8 +20,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists # See https://gradle.org/release-checksums/ for valid checksums -distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionSha256Sum=1541fa36599e12857140465f3c91a97409b4512501c26f9631fb113e392c5bd1 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From b145c311839d0c88af7a4593af64898d88b5c917 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 10 Sep 2024 14:26:01 +0200 Subject: [PATCH 15/36] Introduce renovate (#267) * Introduce renovate The configuration is relatively simple: automerge minor+patch updates and all GH action updates as soon as the updates are available. Exception is the awssdk, which has daily releases, so it's limited to "once per week". For posterity, the config to let Renovate work against version-branches, restricted to patch-version bumps. * Update renovate config: disable automerge --- .github/renovate.json5 | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/renovate.json5 diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 000000000..4e65d7c56 --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + extends: [ + "config:base", + // Required to not pin dependencies to _exact_ versions (pip) + ":preserveSemverRanges", + "group:monorepos", + "helpers:pinGitHubActionDigests", + ":semanticPrefixFixDepsChoreOthers", + ":dependencyDashboard", + ], + + // Added for posterity how to let Renovate manage version-branches, assuming that release branches + // have the `release/` prefix. + baseBranches: ["main", "/^release\\/.*/"], + additionalBranchPrefix: "{{baseBranch}}/", + commitMessagePrefix: "{{baseBranch}}: ", + + pip_requirements: { + // fileMatch default: (^|/)([\\w-]*)requirements\\.(txt|pip)$ + "fileMatch": ["(^|/)([\\w-]*)requirements.*\\.txt$"], + }, + + packageRules: [ + // Check for updates, do not merge automatically. This rule does effectively nothing, but serves as a + // reference for the kinds of managers used in Polaris. + { + matchManagers: [ + "gradle", "gradle-wrapper", + "pip_requirements", "pip_setup", "poetry", "setup-cfg", + "dockerfile", "devcontainer", "docker-compose", + "github-actions"], + automerge: false, // just be explicit (false is the default) + }, + + // Reduce awssdk update frequency noise (awssdk has daily releases) + { + matchManagers: ["gradle"], + matchPackagePrefixes: ["software.amazon.awssdk"], + extends: ["schedule:weekly"], + }, + + // Turn off major & minor version updates on version-branches + { + matchBaseBranches: ["/^release/.*/"], + matchUpdateTypes: ["major", "minor"], + enabled: false + }, + ], + + // Max 50 PRs in total, 5 per hour. + // Renovate rebases PRs automatically when the renovate bot "visits" (scans) the repo, hence the hourly and + // concurrent PRs limit. + prConcurrentLimit: 50, + prHourlyLimit: 5, +} From d3bfa05839019a5f3257764cdd5360b822615b0d Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 10 Sep 2024 14:27:13 +0200 Subject: [PATCH 16/36] Add Chat Bylaws (#272) * Add Chat Bylaws * Adjust wording * Add header --- CHAT_BYLAWS.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 CHAT_BYLAWS.md diff --git a/CHAT_BYLAWS.md b/CHAT_BYLAWS.md new file mode 100644 index 000000000..431dfa1f3 --- /dev/null +++ b/CHAT_BYLAWS.md @@ -0,0 +1,68 @@ + + +# Apache Polaris - Project Chat Bylaws + +Apache Polaris is currently undergoing Incubation at the Apache Software Foundation. + +## Motivation + +The Polaris project uses a public and open chat service currently provided by Zulip hosted +at https://polaris-catalog.zulipchat.com/. A few rules shall ensure that the chat conforms to the rules and best +practices of the Apache Software Foundation and serves well as a collaboration tool for the project. + +Organizations and other open-source projects that contribute continuously and significantly to Polaris are welcome, but +shall not use the Polaris chat as a vehicle for their own marketing without explicit approval of the project (P)PMC. + +## Code of Conduct + +The [Apache Software Foundation’s Code of Conduct](https://www.apache.org/foundation/policies/conduct.html) applies to +the Polaris project public chat. + +## Governance + +* The Polaris project’s chat tool is not provided by the ASF - the Polaris project (P)PMC members govern and monitor the + chat service. +* Everybody is welcome to join the Polaris project chat. Invites are not needed. +* All users are automatically promoted to “members” (don’t stay in “guests”). +* Polaris project (P)PMC members have “owner” privileges on the Zulip chat. Polaris project committers are granted + “moderator” or “administrator” privileges for moderation purposes. +* This bylaws document shall be published on the project’s web site and linked from relevant public channels. +* Only (P)PMC members are allowed to create new channels. The number of channels shall be limited to #general (user + discussions), #dev (development discussions), #announcements (release announcements), and #noise (GitHub feed). +* Only (P)PMC members are allowed to notify a large number of users (aka @here). + +## 3rd Parties + +Definition: “3rd party” means any (other) open-source project or any commercial vendor or any other organization. + +### 3rd Party Channels + +* 3rd parties who contribute to or use the Polaris project may ask the (P)PMC to get a dedicated public channel, for + example #vendor- or #project-. +* Promotion of 3rd party content is only permitted in these 3rd party channels, and only if the content primarily covers + Polaris. Commercial advertisements in any form are prohibited. +* Interaction with users of 3rd party OSS projects or commercial products should be directed to these 3rd party + channels. + +## Content moderation + +* Any content which violates the Apache “Code of Conduct” will be removed. +* Any commercial advertisements will be removed. +* All other content will not be moderated. From 99d8eb7fe383c6283db3d042e17dec33c3bfbd11 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 10 Sep 2024 14:30:57 +0200 Subject: [PATCH 17/36] Add Maven publication (#190) Adds publishing for Maven artifacts, including signing. Includes most prerequisites to publish Maven artifacts to Maven Central. The build-code parts have been taken from Nessie, including the necessary special treatment of shadow-jars and support to publish a bom. `./gradlew publishToMavenLocal` works out of the box. On top of the Nessie parts, this change can also build a source tarball from using `git archive`, plus some Apache project specific adoptions. Fully signed invocation example, assuming GPG agent (there are alternative ways to provide the GPG key+passphrase): ```bash ./ gradlew \ publishToMavenLocal \ sourceTarball \ -Prelease \ -PuseGpgAgent ``` This change also introduces a `version.txt`, which contains `999-SNAPSHOT`. Using `999-SNAPSHOT` on the `main` branch makes it easier later on to enable multiple major/minor version branches and releases from those. `version.txt` on a major/minor version branch would contain something like `2.1-SNAPSHOT`. With `999-SNAPSHOT` on the `main` branch eliminates the need to update `version.txt` on the `main` branch when a new major version branch is created, eliminating the need to commit and push to multiple branches from a mostly automated release workflow infrastructure. --- .github/workflows/gradle.yml | 5 +- Dockerfile | 6 +- build-logic/build.gradle.kts | 2 + .../src/main/kotlin/polaris-java.gradle.kts | 8 + .../src/main/kotlin/polaris-root.gradle.kts | 23 ++ .../main/kotlin/polaris-shadow-jar.gradle.kts | 40 +++ .../main/kotlin/publishing/MemoizedGitInfo.kt | 85 ++++++ .../publishing/PublishingHelperExtension.kt | 61 +++++ .../publishing/PublishingHelperPlugin.kt | 147 +++++++++++ .../main/kotlin/publishing/configurePom.kt | 241 ++++++++++++++++++ .../src/main/kotlin/publishing/rootProject.kt | 198 ++++++++++++++ .../src/main/kotlin/publishing/shadowPub.kt | 125 +++++++++ .../src/main/kotlin/publishing/util.kt | 109 ++++++++ build.gradle.kts | 33 +++ gradle.properties | 1 - gradle/baselibs.versions.toml | 2 + gradle/libs.versions.toml | 1 - polaris-core/build.gradle.kts | 51 ++-- polaris-service/build.gradle.kts | 80 +++--- settings.gradle.kts | 4 + version.txt | 1 + 21 files changed, 1158 insertions(+), 65 deletions(-) create mode 100644 build-logic/src/main/kotlin/polaris-shadow-jar.gradle.kts create mode 100644 build-logic/src/main/kotlin/publishing/MemoizedGitInfo.kt create mode 100644 build-logic/src/main/kotlin/publishing/PublishingHelperExtension.kt create mode 100644 build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt create mode 100644 build-logic/src/main/kotlin/publishing/configurePom.kt create mode 100644 build-logic/src/main/kotlin/publishing/rootProject.kt create mode 100644 build-logic/src/main/kotlin/publishing/shadowPub.kt create mode 100644 build-logic/src/main/kotlin/publishing/util.kt create mode 100644 version.txt diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 97ce4e2d7..288716fb5 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -58,7 +58,10 @@ jobs: - name: Check formatting run: ./gradlew check - + + - name: Check Maven publication + run: ./gradlew publishToMavenLocal sourceTarball + - name: Build with Gradle Wrapper run: ./gradlew test diff --git a/Dockerfile b/Dockerfile index 8fc73438c..0bc4f54b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,13 +31,13 @@ WORKDIR /app RUN rm -rf build # Build the rest catalog -RUN ./gradlew --no-daemon --info -PeclipseLink=$ECLIPSELINK clean shadowJar startScripts +RUN ./gradlew --no-daemon --info -PeclipseLink=$ECLIPSELINK clean prepareDockerDist FROM registry.access.redhat.com/ubi9/openjdk-21-runtime:1.20-2.1721752928 WORKDIR /app -COPY --from=build /app/polaris-service/build/libs/polaris-service-all.jar /app/lib/polaris-service-all.jar +COPY --from=build /app/polaris-service/build/docker-dist/bin /app/bin +COPY --from=build /app/polaris-service/build/docker-dist/lib /app/lib COPY --from=build /app/polaris-server.yml /app -COPY --from=build /app/polaris-service/build/scripts/polaris-service /app/bin/polaris-service EXPOSE 8181 diff --git a/build-logic/build.gradle.kts b/build-logic/build.gradle.kts index b61156e96..eade127b1 100644 --- a/build-logic/build.gradle.kts +++ b/build-logic/build.gradle.kts @@ -24,5 +24,7 @@ dependencies { implementation(baselibs.errorprone) implementation(baselibs.idea.ext) implementation(baselibs.license.report) + implementation(baselibs.nexus.publish) + implementation(baselibs.shadow) implementation(baselibs.spotless) } diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts index 3690e1a95..7c5aebac5 100644 --- a/build-logic/src/main/kotlin/polaris-java.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts @@ -21,6 +21,7 @@ import net.ltgt.gradle.errorprone.errorprone import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.testing.Test import org.gradle.kotlin.dsl.named +import publishing.PublishingHelperPlugin plugins { id("jacoco") @@ -30,6 +31,8 @@ plugins { id("net.ltgt.errorprone") } +apply() + tasks.withType(JavaCompile::class.java).configureEach { options.compilerArgs.addAll(listOf("-Xlint:unchecked", "-Xlint:deprecation")) options.errorprone.disableAllWarnings = true @@ -84,6 +87,11 @@ spotless { dependencies { errorprone(versionCatalogs.named("libs").findLibrary("errorprone").get()) } +java { + withJavadocJar() + withSourcesJar() +} + tasks.withType().configureEach { val opt = options as CoreJavadocOptions // don't spam log w/ "warning: no @param/@return" diff --git a/build-logic/src/main/kotlin/polaris-root.gradle.kts b/build-logic/src/main/kotlin/polaris-root.gradle.kts index ad0211c28..40aa3c494 100644 --- a/build-logic/src/main/kotlin/polaris-root.gradle.kts +++ b/build-logic/src/main/kotlin/polaris-root.gradle.kts @@ -20,12 +20,16 @@ import org.jetbrains.gradle.ext.copyright import org.jetbrains.gradle.ext.encodings import org.jetbrains.gradle.ext.settings +import publishing.PublishingHelperExtension +import publishing.PublishingHelperPlugin plugins { id("com.diffplug.spotless") id("org.jetbrains.gradle.plugin.idea-ext") } +apply() + spotless { kotlinGradle { ktfmt().googleStyle() @@ -57,3 +61,22 @@ if (System.getProperty("idea.sync.active").toBoolean()) { } } } + +extensions.getByType().apply { + asfProjectName = "polaris" + + mailingLists.addAll("dev", "issues", "commits") + + podlingPpmcAsfIds.addAll( + "anoop", + "ashvin", + "jackye", + "jbonofre", + "russellspitzer", + "snazy", + "takidau", + "vvcephei" + ) + podlingMentorsAsfIds.addAll("bdelacretaz", "blue", "holden", "jbonofre", "yao") + podlingCommitterAsfIds.addAll() +} diff --git a/build-logic/src/main/kotlin/polaris-shadow-jar.gradle.kts b/build-logic/src/main/kotlin/polaris-shadow-jar.gradle.kts new file mode 100644 index 000000000..8ed23bee2 --- /dev/null +++ b/build-logic/src/main/kotlin/polaris-shadow-jar.gradle.kts @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { id("com.gradleup.shadow") } + +val shadowJar = tasks.named("shadowJar") + +shadowJar.configure { + outputs.cacheIf { false } // do not cache uber/shaded jars + archiveClassifier = "" + mergeServiceFiles() +} + +tasks.named("jar").configure { + dependsOn(shadowJar) + archiveClassifier = "raw" +} + +tasks.withType().configureEach { + exclude("META-INF/jandex.idx") + isZip64 = true +} diff --git a/build-logic/src/main/kotlin/publishing/MemoizedGitInfo.kt b/build-logic/src/main/kotlin/publishing/MemoizedGitInfo.kt new file mode 100644 index 000000000..df56de76b --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/MemoizedGitInfo.kt @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.api.java.archives.Attributes +import org.gradle.kotlin.dsl.extra +import java.io.ByteArrayOutputStream +import java.nio.charset.StandardCharsets + +/** + * Container to memoize Git information retrieved via `git` command executions across all Gradle projects. + * Jar release artifacts get some attributes added to the jar manifest, which can be quite useful for released jars. + */ +internal class MemoizedGitInfo { + companion object { + private fun execProc(rootProject: Project, cmd: String, vararg args: Any): String { + val buf = ByteArrayOutputStream() + rootProject.exec { + executable = cmd + args(args.toList()) + standardOutput = buf + } + return buf.toString(StandardCharsets.UTF_8).trim() + } + + fun gitInfo(rootProject: Project, attribs: Attributes) { + val props = gitInfo(rootProject) + attribs.putAll(props) + } + + fun gitInfo(rootProject: Project): Map { + return if (rootProject.extra.has("gitReleaseInfo")) { + @Suppress("UNCHECKED_CAST") + rootProject.extra["gitReleaseInfo"] as Map + } else { + val isRelease = rootProject.hasProperty("release") + val gitHead = execProc(rootProject, "git", "rev-parse", "HEAD") + val gitDescribe = if (isRelease) { + try { + execProc(rootProject, "git", "describe", "--tags") + } catch (e: Exception) { + throw GradleException("'git describe --tags' failed - no Git tag?", e) + } + } else { + execProc(rootProject, "git", "describe", "--always", "--dirty") + } + val timestamp = execProc(rootProject, "date", "+%Y-%m-%d-%H:%M:%S%:z") + val system = execProc(rootProject, "uname", "-a") + val javaVersion = System.getProperty("java.version") + + val info = + mapOf( + "Apache-Polaris-Version" to rootProject.version.toString(), + "Apache-Polaris-Is-Release" to isRelease.toString(), + "Apache-Polaris-Build-Git-Head" to gitHead, + "Apache-Polaris-Build-Git-Describe" to gitDescribe, + "Apache-Polaris-Build-Timestamp" to timestamp, + "Apache-Polaris-Build-System" to system, + "Apache-Polaris-Build-Java-Version" to javaVersion + ) + rootProject.extra["gitReleaseInfo"] = info + return info + } + } + } +} \ No newline at end of file diff --git a/build-logic/src/main/kotlin/publishing/PublishingHelperExtension.kt b/build-logic/src/main/kotlin/publishing/PublishingHelperExtension.kt new file mode 100644 index 000000000..7ec06a830 --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/PublishingHelperExtension.kt @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import org.gradle.api.Project +import org.gradle.api.model.ObjectFactory +import org.gradle.kotlin.dsl.listProperty +import org.gradle.kotlin.dsl.property +import java.io.File +import javax.inject.Inject + +/** + * Gradle plugin extension object for the `PublishingHelperPlugin. Most attributes are likely never changed from the + * default values. + * + * Apache podlings need to specify the PPMC members and committers manually, Apache TLPs don't populate these + * properties. + */ +abstract class PublishingHelperExtension +@Inject +constructor(objectFactory: ObjectFactory, project: Project) +{ + // optional customization of the pom.xml element + val mavenName = objectFactory.property().convention(project.provider { project.name }) + + val licenseUrl = objectFactory.property().convention("https://www.apache.org/licenses/LICENSE-2.0.txt") + + // the following are only relevant on the root project + val asfProjectName = objectFactory.property() + val baseName = objectFactory.property().convention(project.provider { "apache-${asfProjectName.get()}-${project.version}" }) + val distributionDir = objectFactory.directoryProperty().convention(project.layout.buildDirectory.dir("distributions")) + val sourceTarball = objectFactory.fileProperty().convention(project.provider { distributionDir.get().file("${baseName.get()}.tar.gz") }) + val sourceTarballDigest = objectFactory.fileProperty().convention(project.provider { distributionDir.get().file("${baseName.get()}.sha512") }) + + val mailingLists = objectFactory.listProperty(String::class.java).convention(emptyList()) + + // override the list of developers (P)PMC members + committers, necessary for podlings + val podlingPpmcAsfIds = objectFactory.setProperty(String::class.java).convention(emptySet()) + val podlingMentorsAsfIds = objectFactory.setProperty(String::class.java).convention(emptySet()) + val podlingCommitterAsfIds = objectFactory.setProperty(String::class.java).convention(emptySet()) + + fun distributionFile(ext: String): File = + distributionDir.get().file("${baseName.get()}.$ext").asFile +} diff --git a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt new file mode 100644 index 000000000..5dbae0a5b --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import com.github.jengelman.gradle.plugins.shadow.ShadowExtension +import org.gradle.api.* +import javax.inject.Inject +import org.gradle.api.component.SoftwareComponentFactory +import org.gradle.api.publish.PublishingExtension +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.publish.maven.plugins.MavenPublishPlugin +import org.gradle.api.publish.tasks.GenerateModuleMetadata +import org.gradle.kotlin.dsl.* +import org.gradle.plugins.signing.SigningExtension +import org.gradle.plugins.signing.SigningPlugin +import org.gradle.jvm.tasks.Jar + +/** Release-publishing helper plugin to generate publications that pass Sonatype validations, + * generate Apache release source tarball. + * + * The `release` Gradle project property triggers: signed artifacts + jars with Git information. + * The current Git HEAD must point to a Git tag. + * + * The `jarWithGitInfo` Gradle project property triggers: jars with Git information (not necessary + * with `release`). + * + * The task `sourceTarball` (available on the root project) generates a source tarball using + * `git archive`. + * + * The task `releaseEmailTemplate` generates the release-vote email subject + body. Outputs on + * the console and in the `build/distributions/` directory. + * + * Signing tip: If you want to use `gpg-agent`, set the `useGpgAgent` Gradle project property + * + * The following command publishes the project artifacts to your local maven repository, + * generates the source tarball - and uses `gpg-agent` to sign all artifacts and the tarball. + * Note that this requires a Git tag! + * ``` + * ./gradlew publishToMavenLocal sourceTarball -Prelease -PuseGpgAgent + * ``` + * + * You can generate signed artifacts when using the `signArtifacts` project property: + * ``` + * ./gradlew publishToMavenLocal sourceTarball -PsignArtifacts -PuseGpgAgent + * ``` + */ +@Suppress("unused") +class PublishingHelperPlugin +@Inject +constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Plugin { + override fun apply(project: Project): Unit = + project.run { + extensions.create("publishingHelper", PublishingHelperExtension::class.java) + + val isRelease = project.hasProperty("release") + val isSigning = isRelease || project.hasProperty("signArtifacts") + + // Adds Git/Build/System related information to the generated jars, if the `release` project + // property is present. Do not add that information in development builds, so that the + // generated jars are still cacheable for Gradle. + if (isRelease || project.hasProperty("jarWithGitInfo")) { + // Runs `git`, considered expensive, so guarded behind project properties. + tasks.withType().configureEach { + manifest { MemoizedGitInfo.gitInfo(rootProject, attributes) } + } + } + + apply(plugin = "maven-publish") + apply(plugin = "signing") + + // Generate a source tarball for a release to be uploaded to + // https://dist.apache.org/repos/dist/dev//apache--/ + if (project == rootProject) { + configureOnRootProject(project) + } + + if (isSigning) { + plugins.withType().configureEach { + configure { + val signingKey: String? by project + val signingPassword: String? by project + useInMemoryPgpKeys(signingKey, signingPassword) + val publishing = project.extensions.getByType(PublishingExtension::class.java) + afterEvaluate { sign(publishing.publications.getByName("maven")) } + + if (project.hasProperty("useGpgAgent")) { + useGpgCmd() + } + } + } + } + + // Gradle complains when a Gradle module metadata ("pom on steroids") is generated with an + // enforcedPlatform() dependency - but Quarkus requires enforcedPlatform(), so we have to + // allow it. + tasks.withType().configureEach { + suppressedValidationErrors.add("enforced-platform") + } + + plugins.withType().configureEach { + configure { + publications { + register("maven") { + val mavenPublication = this + afterEvaluate { + // This MUST happen in an 'afterEvaluate' to ensure that the Shadow*Plugin has + // been applied. + if (project.extensions.findByType(ShadowExtension::class.java) != null) { + configureShadowPublishing(project, mavenPublication, softwareComponentFactory) + } else { + from(components.firstOrNull { c -> c.name == "javaPlatform" || c.name == "java" }) + } + + suppressPomMetadataWarningsFor("testFixturesApiElements") + suppressPomMetadataWarningsFor("testFixturesRuntimeElements") + + mavenPublication.groupId = "$group" + mavenPublication.version = project.version.toString() + } + + tasks.named("generatePomFileForMavenPublication").configure { + configurePom(project, mavenPublication, this) + } + } + } + } + } + } +} + diff --git a/build-logic/src/main/kotlin/publishing/configurePom.kt b/build-logic/src/main/kotlin/publishing/configurePom.kt new file mode 100644 index 000000000..e33cbf48d --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/configurePom.kt @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import groovy.util.Node +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.artifacts.component.ModuleComponentSelector +import org.gradle.api.publish.maven.MavenPom +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.internal.extensions.stdlib.capitalized + +/** + * Configures the content of the generated `pom.xml` files. + * + * For all projects except the root project, the pom gets the ``, ``, ``, `` + * elements and fixes dependencies in `` to be consumable by Maven. + * + * The root project generates the parent pom, containing all the necessary elements to pass Sonatype validation + * and some more information like `` and ``. Most of the information is taken from publicly + * consumable Apache project information from `https://projects.apache.org/json/projects/>.json`. + * `` contains all (P)PMC members and committers from that project info JSON, ordered by real name. + * `` is taken from GitHub's `https://api.github.com/repos/apache//contributors?per_page=1000` + * endpoint to give all contributors credit, ordered by number of contributions (as returned by that endpoint). + */ +internal fun configurePom(project: Project, mavenPublication: MavenPublication, task: Task) = mavenPublication.run { + val e = project.extensions.getByType(PublishingHelperExtension::class.java) + + pom { + if (project != project.rootProject) { + name.set(e.mavenName.get()) + description.set(project.description) + + // Add the license to every pom to make it easier for downstream project to retrieve the license. + licenses { + license { + name.set("Apache-2.0") // SPDX identifier + url.set(e.licenseUrl.get()) + } + } + + withXml { + val projectNode = asNode() + + val parentNode = projectNode.appendNode("parent") + val parent = project.parent!! + parentNode.appendNode("groupId", parent.group) + parentNode.appendNode("artifactId", parent.name) + parentNode.appendNode("version", parent.version) + + addMissingMandatoryDependencyVersions(project, projectNode) + } + } else { + val mavenPom = this + + task.doFirst { + val asfName = e.asfProjectName.get() + val (asfPrj, fromPodlings) = fetchAsfProject(asfName) + + val asfProjectName = asfPrj["name"] as String + + mavenPom.name.set(asfProjectName) + mavenPom.description.set(asfPrj["description"] as String) + + inceptionYear.set((asfPrj["created"] ?: asfPrj["started"]!!).toString().replace("(\\d+)-.*", "\\1")) + url.set(asfPrj["homepage"] as String) + organization { + name.set("The Apache Software Foundation") + url.set("https://www.apache.org/") + } + licenses { + license { + name.set("Apache-2.0") // SPDX identifier + url.set(e.licenseUrl.get()) + } + } + mailingLists { + e.mailingLists.get().forEach { ml -> + mailingList { + name.set("${ml.capitalized()} Mailing List") + subscribe.set("$ml-subscribe@$asfName.apache.org") + unsubscribe.set("$ml-ubsubscribe@$asfName.apache.org") + post.set("$ml@$asfName.apache.org") + archive.set("https://lists.apache.org/list.html?$ml@$asfName.apache.org") + } + } + } + scm { + val codeRepo: String = if (asfPrj.contains("repository")) { + val repos: List = unsafeCast(asfPrj["repository"]) as List + repos[0] + } else { + "https://github.com/apache/$asfName.git" + } + connection.set("scm:git:$codeRepo") + developerConnection.set("scm:git:$codeRepo") + url.set("$codeRepo/tree/main") + tag.set("main") + } + issueManagement { + url.set(if (asfPrj.contains("repository")) { + asfPrj["bug-database"] as String + } else { + "https://github.com/apache/$asfName/issues" + }) + } + addDevelopersToPom(mavenPom, asfName, e, fromPodlings) + addContributorsToPom(mavenPom, asfName, asfProjectName) + } + } + } +} + +/** + * Adds contributors as returned by GitHub, in descending `contributions` order. + */ +fun addContributorsToPom(mavenPom: MavenPom, asfName: String, asfProjectName: String) = mavenPom.run { + contributors { + val contributors: List>? = + parseJson("https://api.github.com/repos/apache/$asfName/contributors?per_page=1000") + if (contributors != null) { + contributors.filter { contributor -> + contributor["type"] == "User" + }.forEach { contributor -> + contributor { + name.set(contributor["login"] as String) + url.set(contributor["url"] as String) + organization.set("$asfProjectName, GitHub contributors") + organizationUrl.set("https://github.com/apache/$asfName") + } + } + } + } +} + +/** + * Adds Apache (P)PMC members + committers, in `name` order. + */ +fun addDevelopersToPom(mavenPom: MavenPom, asfName: String, e: PublishingHelperExtension, fromPodlings: Boolean) = mavenPom.run { + developers { + // Cannot use check the 'groups' for podlings, because the (only) podling's group + // references the mentors, not the PPMC members/committers. There seems to be no + // way to automatically fetch the committers + PPMC members for a podling, except + // maybe + val people: Map> = + parseJson("https://projects.apache.org/json/foundation/people.json")!! + val filteredPeople: List>> + val pmc: (Pair>) -> Boolean + val mentor: (Pair>) -> Boolean + val pmcRole: String + if (!fromPodlings) { + val asfNamePmc = "$asfName-pmc" + filteredPeople = people.filter { entry -> + val groups: List = unsafeCast(entry.value["groups"]) + groups.any { it == asfName || it == asfNamePmc } + }.toList() + pmc = {(_, info) -> + val groups: List = unsafeCast(info["groups"]) + groups.contains(asfNamePmc) + } + pmcRole = "PMC Member" + mentor = {(_, _) -> false} + } else { + val podlingPpmc = e.podlingPpmcAsfIds.get() + val podlingMentors = e.podlingMentorsAsfIds.get() + filteredPeople = (e.podlingCommitterAsfIds.get() + podlingPpmc + podlingMentors).map { id -> + val info = people[id] + if (info == null) { + throw GradleException("Person with ASF id '%id' not found in people.json".format(id)) + } + Pair(id, info) + } + pmc = {(id, _) -> podlingPpmc.contains(id) || podlingMentors.contains(id)} + pmcRole = "PPMC Member" + mentor = {(id, _) -> podlingMentors.contains(id)} + } + + val sortedPeople = filteredPeople.sortedBy { (id, info) -> "${info["name"] as String}_$id" } + + sortedPeople.forEach { (id, info) -> + developer { + this.id.set(id) + this.name.set(info["name"] as String) + this.organization.set("Apache Software Foundation") + this.email.set("$id@apache.org") + this.roles.add("Committer") + if (pmc.invoke(Pair(id, info))) { + this.roles.add(pmcRole) + } + if (mentor.invoke(Pair(id, info))) { + this.roles.add("Mentor") + } + } + } + } +} + +/** + * Scans the generated `pom.xml` for `` in `` that do not have a + * `` and adds one, if possible. Maven kinda requires `` tags there, even if the + * `` without a `` is a bom and that bom's version is available transitively. + */ +fun addMissingMandatoryDependencyVersions(project: Project, projectNode: Node) { + xmlNode(xmlNode(projectNode, "dependencyManagement"), "dependencies")?.children()?.forEach { + val dependency = it as Node + if (xmlNode(dependency, "version") == null) { + val depGroup = xmlNode(dependency, "groupId")!!.text() + val depName = xmlNode(dependency, "artifactId")!!.text() + + var depResult = + findDependency(project.configurations.findByName("runtimeClasspath"), depGroup, depName) + if (depResult == null) { + depResult = + findDependency(project.configurations.findByName("testRuntimeClasspath"), depGroup, depName) + } + + if (depResult != null) { + val req = depResult.requested as ModuleComponentSelector + dependency.appendNode("version", req.version) + } + } + } +} diff --git a/build-logic/src/main/kotlin/publishing/rootProject.kt b/build-logic/src/main/kotlin/publishing/rootProject.kt new file mode 100644 index 000000000..3d14a1cbf --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/rootProject.kt @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import io.github.gradlenexus.publishplugin.NexusPublishExtension +import io.github.gradlenexus.publishplugin.NexusPublishPlugin +import io.github.gradlenexus.publishplugin.internal.StagingRepositoryDescriptorRegistryBuildService +import org.gradle.api.GradleException +import org.gradle.api.Project +import org.gradle.api.services.BuildServiceRegistration +import org.gradle.kotlin.dsl.apply +import org.gradle.kotlin.dsl.named +import org.gradle.kotlin.dsl.register +import org.gradle.plugins.signing.Sign + +/** + * Configures Apache project specific publishing tasks on the root project, for example the source-tarball related tasks. + */ +internal fun configureOnRootProject(project: Project) = project.run { + apply() + + val isRelease = project.hasProperty("release") + val isSigning = isRelease || project.hasProperty("signArtifacts") + + val sourceTarball = tasks.register("sourceTarball") + sourceTarball.configure(){ + group = "build" + description = "Generate a source tarball for a release to be uploaded to dist.apache.org/repos/dist" + + doFirst { + val e = project.extensions.getByType(PublishingHelperExtension::class.java) + mkdir(e.distributionDir) + exec { + executable = "git" + args("archive", "--prefix=${e.baseName.get()}/", "--format=tar.gz", "--output=${e.sourceTarball.get().asFile.relativeTo(projectDir)}", "HEAD") + workingDir(project.projectDir) + } + } + } + + val digestSourceTarball = tasks.register("digestSourceTarball") + digestSourceTarball.configure { + mustRunAfter(sourceTarball) + + doFirst { + val e = project.extensions.getByType(PublishingHelperExtension::class.java) + generateDigest(e.sourceTarball.get().asFile, e.sourceTarballDigest.get().asFile, "SHA-512") + } + } + + sourceTarball.configure { + finalizedBy(digestSourceTarball) + } + + if (isSigning) { + val signSourceTarball = tasks.register("signSourceTarball") + signSourceTarball.configure { + mustRunAfter(sourceTarball) + doFirst { + val e = project.extensions.getByType(PublishingHelperExtension::class.java) + sign(e.sourceTarball.get().asFile) + } + } + sourceTarball.configure { + finalizedBy(signSourceTarball) + } + } + + val releaseEmailTemplate = tasks.register("releaseEmailTemplate") + releaseEmailTemplate.configure { + group = "publishing" + description = "Generate release-vote email subject + body, including the staging repository URL, if run during the Maven release." + + mustRunAfter("initializeApacheStagingRepository") + + doFirst { + val e = project.extensions.getByType(PublishingHelperExtension::class.java) + val asfName = e.asfProjectName.get() + + val gitInfo = MemoizedGitInfo.gitInfo(rootProject) + val gitCommitId = gitInfo["Apache-Polaris-Build-Git-Head"] + + val repos = project.extensions.getByType(NexusPublishExtension::class.java).repositories + val repo = repos.iterator().next() + + val stagingRepositoryUrlRegistryRegistration = gradle.sharedServices.registrations.named>("stagingRepositoryUrlRegistry") + val staginRepoUrl = if (stagingRepositoryUrlRegistryRegistration.isPresent) { + val stagingRepositoryUrlRegistryBuildServiceRegistration = stagingRepositoryUrlRegistryRegistration.get() + val stagingRepositoryUrlRegistryService = stagingRepositoryUrlRegistryBuildServiceRegistration.getService() + if (stagingRepositoryUrlRegistryService.isPresent) { + val registry = stagingRepositoryUrlRegistryService.get().registry + try { + val stagingRepoDesc = registry.get(repo.name) + val stagingRepoId = stagingRepoDesc.stagingRepositoryId + "https://repository.apache.org/content/repositories/$stagingRepoId/" + } catch (e: IllegalStateException) { + "NO STAGING REPOSITORY ($e)" + } + } else { + "NO STAGING REPOSITORY (no registry service) !!" + } + } else { + "NO STAGING REPOSITORY (no build service) !!" + } + + val (asfPrj, _) = fetchAsfProject(asfName) + val asfProjectName = asfPrj["name"] as String + + val versionNoRc = version.toString().replace("-rc-?[0-9]+".toRegex(), "") + + val subjectFile = e.distributionFile("vote-email-subject.txt").relativeTo(projectDir) + val bodyFile = e.distributionFile("vote-email-body.txt").relativeTo(projectDir) + + val emailSubject = "[VOTE] Release $asfProjectName $version" + subjectFile.writeText(emailSubject) + + val emailBody = """ + Hi everyone, + + I propose that we release the following RC as the official + $asfProjectName $versionNoRc release. + + The commit ID is 229d8f6fcd109e6c8943ea7cbb41dab746c6d0ed + * This corresponds to the tag: apache-$asfName-$version + * https://github.com/apache/$asfName/commits/apache-$asfName-$version + * https://github.com/apache/$asfName/tree/$gitCommitId + + The release tarball, signature, and checksums are here: + * https://dist.apache.org/repos/dist/dev/$asfName/apache-$asfName-$version + + You can find the KEYS file here: + * https://dist.apache.org/repos/dist/dev/$asfName/KEYS + + Convenience binary artifacts are staged on Nexus. The Maven repository URL is: + * $staginRepoUrl + + Please download, verify, and test. + + Please vote in the next 72 hours. + + [ ] +1 Release this as Apache Iceberg 1.6.0 + [ ] +0 + [ ] -1 Do not release this because... + + Only PMC members have binding votes, but other community members are + encouraged to cast non-binding votes. This vote will pass if there are + 3 binding +1 votes and more binding +1 votes than -1 votes. + + Thanks + Regards + """ + + logger.lifecycle(""" + + + The email for your release vote mail: + ------------------------------------- + + Suggested subject: (also in file $subjectFile) + + $emailSubject + + Suggested body: (also in file $bodyFile) + + $emailBody + + """.trimIndent()) + bodyFile.writeText(emailBody.trimIndent()) + } + } + + if (isRelease) { + sourceTarball.configure { + finalizedBy(releaseEmailTemplate) + } + } + + afterEvaluate { + tasks.named("closeApacheStagingRepository") { mustRunAfter(releaseEmailTemplate) } + } +} diff --git a/build-logic/src/main/kotlin/publishing/shadowPub.kt b/build-logic/src/main/kotlin/publishing/shadowPub.kt new file mode 100644 index 000000000..4fb78410e --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/shadowPub.kt @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import groovy.util.Node +import groovy.util.NodeList +import org.gradle.api.Project +import org.gradle.api.artifacts.ConfigurationVariant +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.attributes.Bundling +import org.gradle.api.attributes.Category +import org.gradle.api.attributes.LibraryElements +import org.gradle.api.attributes.Usage +import org.gradle.api.component.SoftwareComponentFactory +import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.api.publish.maven.MavenPublication + + +/** + * "Proper" publication of shadow-jar instead of the "main" jar, with "the right" Gradle's module + * metadata that refers to the shadow-jar instead of the "main" jar, which is not published by + * Nessie. + * + * Pieces of this function are taken from the `Java(Base)Plugin` and `ShadowExtension`. + */ +internal fun configureShadowPublishing(project: Project, mavenPublication: MavenPublication, softwareComponentFactory: SoftwareComponentFactory) = project.run { + fun isPublishable(element: ConfigurationVariant): Boolean { + for (artifact in element.artifacts) { + if (JavaBasePlugin.UNPUBLISHABLE_VARIANT_ARTIFACTS.contains(artifact.type)) { + return false + } + } + return true + } + + val shadowJar = project.tasks.named("shadowJar") + + val shadowApiElements = + project.configurations.create("shadowApiElements") { + isCanBeConsumed = true + isCanBeResolved = false + attributes { + attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage::class.java, Usage.JAVA_API)) + attribute( + Category.CATEGORY_ATTRIBUTE, + project.objects.named(Category::class.java, Category.LIBRARY) + ) + attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + project.objects.named(LibraryElements::class.java, LibraryElements.JAR) + ) + attribute( + Bundling.BUNDLING_ATTRIBUTE, + project.objects.named(Bundling::class.java, Bundling.SHADOWED) + ) + } + outgoing.artifact(shadowJar) + } + + val component = softwareComponentFactory.adhoc("shadow") + component.addVariantsFromConfiguration(shadowApiElements) { + if (isPublishable(configurationVariant)) { + mapToMavenScope("compile") + } else { + skip() + } + } + // component.addVariantsFromConfiguration(configurations.getByName("runtimeElements")) { + component.addVariantsFromConfiguration( + project.configurations.getByName("shadowRuntimeElements") + ) { + if (isPublishable(configurationVariant)) { + mapToMavenScope("runtime") + } else { + skip() + } + } + // Sonatype requires the javadoc and sources jar to be present, but the + // Shadow extension does not publish those. + component.addVariantsFromConfiguration(project.configurations.getByName("javadocElements")) {} + component.addVariantsFromConfiguration(project.configurations.getByName("sourcesElements")) {} + mavenPublication.from(component) + + // This a replacement to add dependencies to the pom, if necessary. Equivalent to + // 'shadowExtension.component(mavenPublication)', which we cannot use. + + mavenPublication.pom { + withXml { + val node = asNode() + val depNode = node.get("dependencies") + val dependenciesNode = + if ((depNode as NodeList).isNotEmpty()) depNode[0] as Node + else node.appendNode("dependencies") + project.configurations.getByName("shadow").allDependencies.forEach { + @Suppress("DEPRECATION") + if ( + (it is ProjectDependency) || it !is org.gradle.api.artifacts.SelfResolvingDependency + ) { + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", it.group) + dependencyNode.appendNode("artifactId", it.name) + dependencyNode.appendNode("version", it.version) + dependencyNode.appendNode("scope", "runtime") + } + } + } + } +} diff --git a/build-logic/src/main/kotlin/publishing/util.kt b/build-logic/src/main/kotlin/publishing/util.kt new file mode 100644 index 000000000..ee5919ce7 --- /dev/null +++ b/build-logic/src/main/kotlin/publishing/util.kt @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package publishing + +import groovy.json.JsonException +import groovy.json.JsonSlurper +import groovy.util.Node +import groovy.util.NodeList +import groovy.xml.XmlSlurper +import groovy.xml.slurpersupport.GPathResult +import org.gradle.api.GradleException +import org.gradle.api.artifacts.Configuration +import org.gradle.api.artifacts.component.ModuleComponentSelector +import org.gradle.api.artifacts.result.DependencyResult +import java.io.File +import java.io.FileNotFoundException +import java.net.URI +import java.security.MessageDigest + +internal fun findDependency( + config: Configuration?, + depGroup: String, + depName: String +): DependencyResult? { + if (config != null) { + val depResult = + config.incoming.resolutionResult.allDependencies.find { depResult -> + val req = depResult.requested + if (req is ModuleComponentSelector) req.group == depGroup && req.module == depName + else false + } + return depResult + } + return null +} + +internal fun xmlNode(node: Node?, child: String): Node? { + val found = node?.get(child) + if (found is NodeList) { + if (found.isNotEmpty()) { + return found[0] as Node + } + } + return null +} + +internal fun generateDigest(input: File, output: File, algorithm: String) { + val md = MessageDigest.getInstance(algorithm) + input.inputStream().use { + val buffered = it.buffered(8192) + val buf = ByteArray(8192) + var rd: Int + while (true) { + rd = buffered.read(buf) + if (rd == -1) break + md.update(buf, 0, rd) + } + + output.writeText(md.digest().joinToString(separator = "") { eachByte -> "%02x".format(eachByte) }) + } +} + +internal fun fetchAsfProject(asfName: String): Pair, Boolean> { + val tlpPrj: Map? = parseJson("https://projects.apache.org/json/projects/$asfName.json") + return if (tlpPrj != null) { + Pair(tlpPrj, false) + } else { + val podlings: Map> = parseJson("https://projects.apache.org/json/foundation/podlings.json")!! + val podling = podlings[asfName] + if (podling == null) { + throw GradleException("Neither a project nor a podling for $asfName could be found at Apache") + } + Pair(podling, true) + } +} + +internal fun unsafeCast(o: Any?): T { + @Suppress("UNCHECKED_CAST") + return o as T +} + +internal fun parseJson(url: String): T? { + try { + @Suppress("UNCHECKED_CAST") + return JsonSlurper().parse(URI(url).toURL()) as T + } catch (e: JsonException) { + if (e.cause is FileNotFoundException) { + return null + } + throw e + } +} diff --git a/build.gradle.kts b/build.gradle.kts index 0b58abc8c..12828e156 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,6 +17,7 @@ * under the License. */ +import java.net.URI import org.nosphere.apache.rat.RatTask buildscript { @@ -108,3 +109,35 @@ tasks.named("rat").configure { excludes.add("**/*.env*") } + +// Pass environment variables: +// ORG_GRADLE_PROJECT_apacheUsername +// ORG_GRADLE_PROJECT_apachePassword +// OR in ~/.gradle/gradle.properties set +// apacheUsername +// apachePassword +// Call targets: +// publishToApache +// closeApacheStagingRepository +// releaseApacheStagingRepository +// or closeAndReleaseApacheStagingRepository +// +// Username is your ASF ID +// Password: your ASF LDAP password - or better: a token generated via +// https://repository.apache.org/ +nexusPublishing { + transitionCheckOptions { + // default==60 (10 minutes), wait up to 120 minutes + maxRetries = 720 + // default 10s + delayBetween = java.time.Duration.ofSeconds(10) + } + + repositories { + register("apache") { + nexusUrl = URI.create("https://repository.apache.org/service/local/") + snapshotRepositoryUrl = + URI.create("https://repository.apache.org/content/repositories/snapshots/") + } + } +} diff --git a/gradle.properties b/gradle.properties index a934ca712..9e1de428d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,7 +17,6 @@ # under the License. # group=org.apache.polaris -version=1.0.0 # enable the Gradle build cache org.gradle.caching=true diff --git a/gradle/baselibs.versions.toml b/gradle/baselibs.versions.toml index 5391dcd62..72da54e13 100644 --- a/gradle/baselibs.versions.toml +++ b/gradle/baselibs.versions.toml @@ -21,4 +21,6 @@ errorprone = { module = "net.ltgt.gradle:gradle-errorprone-plugin", version = "4.0.1" } idea-ext = { module = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext", version = "1.1.8" } license-report = { module = "com.github.jk1:gradle-license-report", version = "2.8" } +nexus-publish = { module = "io.github.gradle-nexus:publish-plugin", version = "2.0.0" } +shadow = { module = "com.gradleup.shadow:shadow-gradle-plugin", version = "8.3.0" } spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = "6.25.0" } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 49a1106a7..8f83ef160 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -76,4 +76,3 @@ threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" } openapi-generator = { id = "org.openapi.generator", version = "7.6.0" } rat = { id = "org.nosphere.apache.rat", version = "0.8.1" } spotless = { id = "com.diffplug.spotless", version = "6.25.0" } -shadow = { id = "com.gradleup.shadow", version = "8.3.0" } diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts index c1850b313..b07a18767 100644 --- a/polaris-core/build.gradle.kts +++ b/polaris-core/build.gradle.kts @@ -127,32 +127,33 @@ dependencies { openApiValidate { inputSpec = "$rootDir/spec/polaris-management-service.yml" } -tasks.register("generatePolarisService").configure { - inputSpec = "$rootDir/spec/polaris-management-service.yml" - generatorName = "jaxrs-resteasy" - outputDir = "$projectDir/build/generated" - modelPackage = "org.apache.polaris.core.admin.model" - ignoreFileOverride = "$rootDir/.openapi-generator-ignore" - removeOperationIdPrefix = true - templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "false") - globalProperties.put("models", "") - globalProperties.put("apiDocs", "false") - globalProperties.put("modelTests", "false") - configOptions.put("useBeanValidation", "true") - configOptions.put("sourceFolder", "src/main/java") - configOptions.put("useJakartaEe", "true") - configOptions.put("generateBuilders", "true") - configOptions.put("generateConstructorWithAllArgs", "true") - additionalProperties.put("apiNamePrefix", "Polaris") - additionalProperties.put("apiNameSuffix", "Api") - additionalProperties.put("metricsPrefix", "polaris") - serverVariables = mapOf("basePath" to "api/v1") - - doFirst { delete(outputDir.get()) } -} +val generatePolarisService by + tasks.registering(GenerateTask::class) { + inputSpec = "$rootDir/spec/polaris-management-service.yml" + generatorName = "jaxrs-resteasy" + outputDir = "$projectDir/build/generated" + modelPackage = "org.apache.polaris.core.admin.model" + ignoreFileOverride = "$rootDir/.openapi-generator-ignore" + removeOperationIdPrefix = true + templateDir = "$rootDir/server-templates" + globalProperties.put("apis", "false") + globalProperties.put("models", "") + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") + configOptions.put("useBeanValidation", "true") + configOptions.put("sourceFolder", "src/main/java") + configOptions.put("useJakartaEe", "true") + configOptions.put("generateBuilders", "true") + configOptions.put("generateConstructorWithAllArgs", "true") + additionalProperties.put("apiNamePrefix", "Polaris") + additionalProperties.put("apiNameSuffix", "Api") + additionalProperties.put("metricsPrefix", "polaris") + serverVariables = mapOf("basePath" to "api/v1") + } -tasks.named("compileJava").configure { dependsOn("generatePolarisService") } +listOf("sourcesJar", "compileJava").forEach { task -> + tasks.named(task) { dependsOn(generatePolarisService) } +} sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts index e0f990ae3..c0cfa60f7 100644 --- a/polaris-service/build.gradle.kts +++ b/polaris-service/build.gradle.kts @@ -21,10 +21,10 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { - alias(libs.plugins.shadow) alias(libs.plugins.openapi.generator) id("polaris-server") id("polaris-license-report") + id("polaris-shadow-jar") id("application") } @@ -174,34 +174,35 @@ openApiGenerate { ) } -tasks.register("generatePolarisService").configure { - inputSpec = "$rootDir/spec/polaris-management-service.yml" - generatorName = "jaxrs-resteasy" - outputDir = "$projectDir/build/generated" - apiPackage = "org.apache.polaris.service.admin.api" - modelPackage = "org.apache.polaris.core.admin.model" - ignoreFileOverride = "$rootDir/.openapi-generator-ignore" - removeOperationIdPrefix = true - templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "") - globalProperties.put("models", "false") - globalProperties.put("apiDocs", "false") - globalProperties.put("modelTests", "false") - configOptions.put("useBeanValidation", "true") - configOptions.put("sourceFolder", "src/main/java") - configOptions.put("useJakartaEe", "true") - configOptions.put("generateBuilders", "true") - configOptions.put("generateConstructorWithAllArgs", "true") - additionalProperties.put("apiNamePrefix", "Polaris") - additionalProperties.put("apiNameSuffix", "Api") - additionalProperties.put("metricsPrefix", "polaris") - serverVariables.put("basePath", "api/v1") +val generatePolarisService by + tasks.registering(GenerateTask::class) { + inputSpec = "$rootDir/spec/polaris-management-service.yml" + generatorName = "jaxrs-resteasy" + outputDir = "$projectDir/build/generated" + apiPackage = "org.apache.polaris.service.admin.api" + modelPackage = "org.apache.polaris.core.admin.model" + ignoreFileOverride = "$rootDir/.openapi-generator-ignore" + removeOperationIdPrefix = true + templateDir = "$rootDir/server-templates" + globalProperties.put("apis", "") + globalProperties.put("models", "false") + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") + configOptions.put("useBeanValidation", "true") + configOptions.put("sourceFolder", "src/main/java") + configOptions.put("useJakartaEe", "true") + configOptions.put("generateBuilders", "true") + configOptions.put("generateConstructorWithAllArgs", "true") + additionalProperties.put("apiNamePrefix", "Polaris") + additionalProperties.put("apiNameSuffix", "Api") + additionalProperties.put("metricsPrefix", "polaris") + serverVariables.put("basePath", "api/v1") + } - doFirst { delete(outputDir.get()) } +listOf("sourcesJar", "compileJava").forEach { task -> + tasks.named(task) { dependsOn("openApiGenerate", generatePolarisService) } } -tasks.named("compileJava").configure { dependsOn("openApiGenerate", "generatePolarisService") } - sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } @@ -230,13 +231,24 @@ tasks.named("jar") { manifest { attributes["Main-Class"] = "org.apache.polaris.service.PolarisApplication" } } -tasks.named("shadowJar") { - manifest { attributes["Main-Class"] = "org.apache.polaris.service.PolarisApplication" } - archiveVersion.set("") - mergeServiceFiles() - isZip64 = true -} +val shadowJar = + tasks.named("shadowJar") { + manifest { attributes["Main-Class"] = "org.apache.polaris.service.PolarisApplication" } + mergeServiceFiles() + isZip64 = true + finalizedBy("startScripts") + } -tasks.named("startScripts") { classpath = files("polaris-service-all.jar") } +val startScripts = + tasks.named("startScripts") { + classpath = files(provider { shadowJar.get().archiveFileName }) + } + +tasks.register("prepareDockerDist") { + into(project.layout.buildDirectory.dir("docker-dist")) + from(startScripts) { into("bin") } + from(shadowJar) { into("lib") } + doFirst { delete(project.layout.buildDirectory.dir("regtest-dist")) } +} -tasks.named("build").configure { dependsOn("shadowJar") } +tasks.named("build").configure { dependsOn("prepareDockerDist") } diff --git a/settings.gradle.kts b/settings.gradle.kts index fad172deb..e23cf56c4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -36,6 +36,8 @@ if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) { rootProject.name = "polaris" +val baseVersion = file("version.txt").readText().trim() + fun loadProperties(file: File): Properties { val props = Properties() file.reader().use { reader -> props.load(reader) } @@ -69,3 +71,5 @@ dependencyResolutionManagement { gradlePluginPortal() } } + +gradle.beforeProject { version = baseVersion } diff --git a/version.txt b/version.txt new file mode 100644 index 000000000..3db3d09d3 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +999-SNAPSHOT \ No newline at end of file From 50da177d4f5f62c097c7c5f0820b88a9e8ddf7a5 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 10 Sep 2024 09:23:20 -0700 Subject: [PATCH 18/36] Add catalog-level override for ALLOW_TABLE_LOCATION_OVERLAP (#264) --- .../polaris/core/PolarisConfiguration.java | 1 + .../service/catalog/BasePolarisCatalog.java | 15 +- .../service/admin/PolarisAuthzTestBase.java | 2 +- .../admin/PolarisOverlappingTableTest.java | 294 ++++++++++++++++++ .../PolarisRestCatalogIntegrationTest.java | 3 +- 5 files changed, 308 insertions(+), 7 deletions(-) create mode 100644 polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java index ee101997d..ea15e9b01 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisConfiguration.java @@ -104,6 +104,7 @@ public static Builder builder() { public static final PolarisConfiguration ALLOW_TABLE_LOCATION_OVERLAP = PolarisConfiguration.builder() .key("ALLOW_TABLE_LOCATION_OVERLAP") + .catalogConfig("allow.overlapping.table.location") .description( "If set to true, allow one table's location to reside within another table's location. " + "This is only enforced within a given namespace.") diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index d27cc82c4..2b1136e6f 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1006,12 +1006,16 @@ private void validateLocationsForTableLike( * configuration of the service */ private void validateNoLocationOverlap( - TableIdentifier identifier, List resolvedNamespace, String location) { + CatalogEntity catalog, + TableIdentifier identifier, + List resolvedNamespace, + String location) { if (callContext .getPolarisCallContext() .getConfigurationStore() .getConfiguration( callContext.getPolarisCallContext(), + catalog, PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP)) { LOGGER.debug("Skipping location overlap validation for identifier '{}'", identifier); } else { // if (entity.getSubType().equals(PolarisEntitySubType.TABLE)) { @@ -1136,7 +1140,7 @@ private void validateNoLocationOverlap( URI existing = URI.create(siblingLocation); if (isUnderParentLocation(target, existing) || isUnderParentLocation(existing, target)) { - throw new org.apache.iceberg.exceptions.BadRequestException( + throw new org.apache.iceberg.exceptions.ForbiddenException( "Unable to create table at location '%s' because it conflicts with existing table or namespace at location '%s'", target, existing); } @@ -1290,7 +1294,9 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { validateLocationsForTableLike(tableIdentifier, dataLocations, resolvedStorageEntity); // also validate that the table location doesn't overlap an existing table dataLocations.forEach( - location -> validateNoLocationOverlap(tableIdentifier, resolvedNamespace, location)); + location -> + validateNoLocationOverlap( + catalogEntity, tableIdentifier, resolvedNamespace, location)); // and that the metadata file points to a location within the table's directory structure if (metadata.metadataFileLocation() != null) { validateMetadataFileInTableDir(tableIdentifier, metadata, catalog); @@ -1503,7 +1509,8 @@ public void doCommit(ViewMetadata base, ViewMetadata metadata) { // If location is changing then we must validate that the requested location is valid // for the storage configuration inherited under this entity's path. validateLocationForTableLike(identifier, metadata.location(), resolvedStorageEntity); - validateNoLocationOverlap(identifier, resolvedNamespace, metadata.location()); + validateNoLocationOverlap( + catalogEntity, identifier, resolvedNamespace, metadata.location()); } Map tableProperties = new HashMap<>(metadata.properties()); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index 4133082a8..19177f079 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -122,7 +122,7 @@ public abstract class PolarisAuthzTestBase { protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; - protected static final Schema SCHEMA = + public static final Schema SCHEMA = new Schema( required(3, "id", Types.IntegerType.get(), "unique ID 🤪"), required(4, "data", Types.StringType.get())); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java new file mode 100644 index 000000000..c30a34558 --- /dev/null +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.admin; + +import static org.apache.polaris.service.admin.PolarisAuthzTestBase.SCHEMA; +import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.polaris.core.PolarisConfiguration; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.PolarisApplication; +import org.apache.polaris.service.config.PolarisApplicationConfig; +import org.apache.polaris.service.test.PolarisConnectionExtension; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +@ExtendWith({DropwizardExtensionsSupport.class, PolarisConnectionExtension.class}) +public class PolarisOverlappingTableTest { + private static final DropwizardAppExtension BASE_EXT = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), + // Bind to random port to support parallelism + ConfigOverride.config("server.applicationConnectors[0].port", "0"), + ConfigOverride.config("server.adminConnectors[0].port", "0"), + // Enforce table location constraints + ConfigOverride.config("featureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION", "false"), + ConfigOverride.config("featureConfiguration.ALLOW_TABLE_LOCATION_OVERLAP", "false")); + + private static final DropwizardAppExtension LAX_EXT = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), + // Bind to random port to support parallelism + ConfigOverride.config("server.applicationConnectors[0].port", "0"), + ConfigOverride.config("server.adminConnectors[0].port", "0"), + // Relax table location constraints + ConfigOverride.config("featureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION", "true"), + ConfigOverride.config("featureConfiguration.ALLOW_TABLE_LOCATION_OVERLAP", "true")); + + private static PolarisConnectionExtension.PolarisToken adminToken; + private static String userToken; + private static String realm; + private static String namespace; + private static final String baseLocation = "file:///tmp/PolarisOverlappingTableTest"; + + private static final CatalogWrapper defaultCatalog = new CatalogWrapper("default"); + private static final CatalogWrapper laxCatalog = new CatalogWrapper("lax"); + private static final CatalogWrapper strictCatalog = new CatalogWrapper("strict"); + + /** Used to define a parameterized test config */ + protected record TestConfig( + DropwizardAppExtension extension, + CatalogWrapper catalogWrapper, + Response.Status response) { + public String catalog() { + return catalogWrapper.catalog; + } + + private String extensionName() { + return (extension + .getConfiguration() + .getConfigurationStore() + .getConfiguration(null, PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP)) + ? "lax" + : "strict"; + } + + /** Extract the first component of the catalog name; e.g. `default` from `default_123_xyz` */ + private String catalogShortName() { + int firstComponentEnd = catalog().indexOf('_'); + if (firstComponentEnd != -1) { + return catalog().substring(0, firstComponentEnd); + } else { + return catalog(); + } + } + + @Override + public String toString() { + return String.format( + "extension=%s, catalog=%s, status=%s", + extensionName(), catalogShortName(), response.toString()); + } + } + + /* Used to wrap a catalog name, so the TestConfig's final `catalog` field can be updated */ + protected static class CatalogWrapper { + public String catalog; + + public CatalogWrapper(String catalog) { + this.catalog = catalog; + } + + @Override + public String toString() { + return catalog; + } + } + + @BeforeEach + public void setup(PolarisConnectionExtension.PolarisToken adminToken) { + userToken = adminToken.token(); + realm = PolarisConnectionExtension.getTestRealm(PolarisServiceImplIntegrationTest.class); + defaultCatalog.catalog = String.format("default_catalog_%s", UUID.randomUUID().toString()); + laxCatalog.catalog = String.format("lax_catalog_%s", UUID.randomUUID().toString()); + strictCatalog.catalog = String.format("strict_catalog_%s", UUID.randomUUID().toString()); + for (var EXT : List.of(BASE_EXT, LAX_EXT)) { + for (var c : List.of(defaultCatalog, laxCatalog, strictCatalog)) { + CatalogProperties.Builder propertiesBuilder = + CatalogProperties.builder() + .setDefaultBaseLocation(String.format("%s/%s", baseLocation, c)); + if (!c.equals(defaultCatalog)) { + propertiesBuilder + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + String.valueOf(c.equals(laxCatalog))) + .addProperty( + PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP.catalogConfig(), + String.valueOf(c.equals(laxCatalog))); + } + StorageConfigInfo config = + FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .build(); + Catalog catalogObject = + new Catalog( + Catalog.TypeEnum.INTERNAL, + c.catalog, + propertiesBuilder.build(), + 1725487592064L, + 1725487592064L, + 1, + config); + try (Response response = + request(EXT, "management/v1/catalogs") + .post(Entity.json(new CreateCatalogRequest(catalogObject)))) { + if (response.getStatus() != Response.Status.CREATED.getStatusCode()) { + throw new IllegalStateException( + "Failed to create catalog: " + response.readEntity(String.class)); + } + } + + namespace = "ns"; + CreateNamespaceRequest createNamespaceRequest = + CreateNamespaceRequest.builder().withNamespace(Namespace.of(namespace)).build(); + try (Response response = + request(EXT, String.format("catalog/v1/%s/namespaces", c)) + .post(Entity.json(createNamespaceRequest))) { + if (response.getStatus() != Response.Status.OK.getStatusCode()) { + throw new IllegalStateException( + "Failed to create namespace: " + response.readEntity(String.class)); + } + } + } + } + } + + private Response createTable( + DropwizardAppExtension extension, String catalog, String location) { + CreateTableRequest createTableRequest = + CreateTableRequest.builder() + .withName("table_" + UUID.randomUUID().toString()) + .withLocation(location) + .withSchema(SCHEMA) + .build(); + String prefix = String.format("catalog/v1/%s/namespaces/%s/tables", catalog, namespace); + try (Response response = request(extension, prefix).post(Entity.json(createTableRequest))) { + return response; + } + } + + private static Invocation.Builder request( + DropwizardAppExtension extension, String prefix) { + return extension + .client() + .target(String.format("http://localhost:%d/api/%s", extension.getLocalPort(), prefix)) + .request("application/json") + .header("Authorization", "Bearer " + userToken) + .header(REALM_PROPERTY_KEY, realm); + } + + private static Stream getTestConfigs() { + return Stream.of( + new TestConfig(BASE_EXT, defaultCatalog, Response.Status.FORBIDDEN), + new TestConfig(BASE_EXT, strictCatalog, Response.Status.FORBIDDEN), + new TestConfig(BASE_EXT, laxCatalog, Response.Status.OK), + new TestConfig(LAX_EXT, defaultCatalog, Response.Status.OK), + new TestConfig(LAX_EXT, strictCatalog, Response.Status.FORBIDDEN), + new TestConfig(LAX_EXT, laxCatalog, Response.Status.OK)); + } + + @ParameterizedTest + @MethodSource("getTestConfigs") + @DisplayName("Test restrictions on table locations") + void testTableLocationRestrictions(TestConfig config) { + // Original table + assertThat( + createTable( + config.extension, + config.catalog(), + String.format("%s/%s/%s/table_1", baseLocation, config.catalog(), namespace))) + .returns(Response.Status.OK.getStatusCode(), Response::getStatus); + + // Unrelated path + assertThat( + createTable( + config.extension, + config.catalog(), + String.format("%s/%s/%s/table_2", baseLocation, config.catalog(), namespace))) + .returns(Response.Status.OK.getStatusCode(), Response::getStatus); + + // Trailing slash makes this not overlap with table_1 + assertThat( + createTable( + config.extension, + config.catalog(), + String.format("%s/%s/%s/table_100", baseLocation, config.catalog(), namespace))) + .returns(Response.Status.OK.getStatusCode(), Response::getStatus); + + // Repeat location + assertThat( + createTable( + config.extension, + config.catalog(), + String.format("%s/%s/%s/table_100", baseLocation, config.catalog(), namespace))) + .returns(config.response.getStatusCode(), Response::getStatus); + + // Parent of existing location + assertThat( + createTable( + config.extension, + config.catalog(), + String.format("%s/%s/%s", baseLocation, config.catalog(), namespace))) + .returns(config.response.getStatusCode(), Response::getStatus); + + // Child of existing location + assertThat( + createTable( + config.extension, + config.catalog(), + String.format( + "%s/%s/%s/table_100/child", baseLocation, config.catalog(), namespace))) + .returns(config.response.getStatusCode(), Response::getStatus); + + // Outside the namespace + assertThat( + createTable( + config.extension, + config.catalog(), + String.format("%s/%s", baseLocation, config.catalog()))) + .returns(config.response.getStatusCode(), Response::getStatus); + + // Outside the catalog + assertThat(createTable(config.extension, config.catalog(), String.format("%s", baseLocation))) + .returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); + } +} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java index c1542001e..449d6811f 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java @@ -47,7 +47,6 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.BadRequestException; import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; @@ -685,7 +684,7 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( .buildTable(TableIdentifier.of(Namespace.of("ns1", "ns1a"), "tbl2"), SCHEMA) .withLocation(catalogBaseLocation + "/ns1/ns1a-override/tbl1-override") .create()) - .isInstanceOf(BadRequestException.class) + .isInstanceOf(ForbiddenException.class) .hasMessageContaining("because it conflicts with existing table or namespace"); } From cc58730a0c626dba574c2577b32081b17b97b7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Wed, 11 Sep 2024 17:36:36 +0200 Subject: [PATCH 19/36] Upgrade to Apache Iceberg 1.6.1 (#7) --- gradle/libs.versions.toml | 2 +- .../service/catalog/PolarisRestCatalogIntegrationTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8f83ef160..c69cca0dc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ [versions] hadoop = "3.4.0" -iceberg = "1.5.0" +iceberg = "1.6.1" dropwizard = "4.0.8" slf4j = "2.0.13" swagger = "1.6.14" diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java index 449d6811f..b2ccb992c 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java @@ -152,7 +152,7 @@ public void before( .getTestMethod() .ifPresent( method -> { - currentCatalogName = method.getName(); + currentCatalogName = method.getName() + UUID.randomUUID(); AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn(TEST_ROLE_ARN) From 1e36b7cd20593aac23898198f5a87d3a3e951053 Mon Sep 17 00:00:00 2001 From: Tyler Akidau Date: Wed, 11 Sep 2024 23:39:50 +0200 Subject: [PATCH 20/36] Add new committer Yufei Gu to CODEOWNERS (#283) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6e69b9850..7b382973c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,4 +17,4 @@ # under the License. # -* @jbonofre @ashvina @anoopj @RussellSpitzer @snazy @vvcephei @takidau @jackye1995 +* @jbonofre @ashvina @anoopj @RussellSpitzer @snazy @vvcephei @takidau @jackye1995 @flyrain From fc6b51892866d0036c2653c824e45df29fafefc7 Mon Sep 17 00:00:00 2001 From: Aihua Xu Date: Wed, 11 Sep 2024 14:40:48 -0700 Subject: [PATCH 21/36] Enable EclipseLink with H2 database in testing (#158) --- .../persistence/eclipselink/build.gradle.kts | 2 +- ...olarisEclipseLinkMetaStoreSessionImpl.java | 7 +-- .../main/resources/META-INF/persistence.xml | 2 +- ...olarisEclipseLinkMetaStoreManagerTest.java | 4 +- .../test/resources/META-INF/persistence.xml | 47 ------------------ polaris-service/build.gradle.kts | 3 ++ .../service/BootstrapRealmsCommand.java | 3 +- .../admin/PolarisOverlappingCatalogTest.java | 8 ++- .../PolarisServiceImplIntegrationTest.java | 5 +- .../PolarisRestCatalogIntegrationTest.java | 8 +-- ...PolarisRestCatalogViewIntegrationTest.java | 8 +-- .../catalog/PolarisSparkIntegrationTest.java | 7 ++- .../ManifestFileCleanupTaskHandlerTest.java | 21 ++++---- .../task/TableCleanupTaskHandlerTest.java | 21 ++++---- .../test/PolarisConnectionExtension.java | 49 ++++++++++++++++++- .../polaris-server-integrationtest.yml | 2 +- 16 files changed, 98 insertions(+), 99 deletions(-) delete mode 100644 extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 12e0affd0..3137a0cc0 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -44,7 +44,7 @@ tasks.register("archiveConf") { archiveFileName = "conf.jar" destinationDirectory = layout.buildDirectory.dir("conf") - from("src/test/resources/META-INF/") { include("persistence.xml") } + from("src/main/resources/META-INF/") { include("persistence.xml") } } tasks.named("test") { dependsOn("archiveConf") } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index dec10d3a4..da6d90055 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -111,7 +111,8 @@ public PolarisEclipseLinkMetaStoreSessionImpl( @NotNull RealmContext realmContext, @Nullable String confFile, @Nullable String persistenceUnitName) { - LOGGER.debug("Create EclipseLink Meta Store Session for {}", realmContext.getRealmIdentifier()); + LOGGER.debug( + "Creating EclipseLink Meta Store Session for realm {}", realmContext.getRealmIdentifier()); emf = createEntityManagerFactory(realmContext, confFile, persistenceUnitName); // init store @@ -153,8 +154,8 @@ private EntityManagerFactory createEntityManagerFactory( prefixUrl = new File(jarPrefixPath).toURI().toURL(); } - LOGGER.info( - "Created a new ClassLoader with the jar {} in classpath to load the config file", + LOGGER.debug( + "Creating a new ClassLoader with the jar {} in classpath to load the config file", prefixUrl); URLClassLoader currentClassLoader = diff --git a/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml b/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml index a364fe06c..913713796 100644 --- a/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml +++ b/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml @@ -35,7 +35,7 @@ NONE + value="jdbc:h2:file:./build/test_data/polaris/{realm}/db"/> diff --git a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java index ea28f8c67..000a909e9 100644 --- a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java @@ -51,7 +51,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { PolarisEclipseLinkStore store = new PolarisEclipseLinkStore(diagServices); PolarisEclipseLinkMetaStoreSessionImpl session = new PolarisEclipseLinkMetaStoreSessionImpl( - store, Mockito.mock(), () -> "realm", null, "polaris-dev"); + store, Mockito.mock(), () -> "realm", null, "polaris"); return new PolarisTestMetaStoreManager( new PolarisMetaStoreManagerImpl(), new PolarisCallContext( @@ -72,7 +72,7 @@ void testCreateStoreSession(String confFile, boolean success) { try { var session = new PolarisEclipseLinkMetaStoreSessionImpl( - store, Mockito.mock(), () -> "realm", confFile, "polaris-dev"); + store, Mockito.mock(), () -> "realm", confFile, "polaris"); assertNotNull(session); assertTrue(success); } catch (Exception e) { diff --git a/extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml b/extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml deleted file mode 100644 index 64952f7c0..000000000 --- a/extension/persistence/eclipselink/src/test/resources/META-INF/persistence.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - org.eclipse.persistence.jpa.PersistenceProvider - org.apache.polaris.core.persistence.models.ModelEntity - org.apache.polaris.core.persistence.models.ModelEntityActive - org.apache.polaris.core.persistence.models.ModelEntityChangeTracking - org.apache.polaris.core.persistence.models.ModelEntityDropped - org.apache.polaris.core.persistence.models.ModelGrantRecord - org.apache.polaris.core.persistence.models.ModelPrincipalSecrets - org.apache.polaris.core.persistence.models.ModelSequenceId - NONE - - - - - - - - - - - \ No newline at end of file diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts index c0cfa60f7..6363842b2 100644 --- a/polaris-service/build.gradle.kts +++ b/polaris-service/build.gradle.kts @@ -109,6 +109,9 @@ dependencies { testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) testRuntimeOnly("org.junit.platform:junit-platform-launcher") + + testRuntimeOnly(project(":polaris-eclipselink")) + testRuntimeOnly(libs.h2) } if (project.properties.get("eclipseLink") == "true") { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java index 3e7b8fc31..aab0e1229 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java @@ -48,8 +48,7 @@ public BootstrapRealmsCommand() { protected void run( Bootstrap bootstrap, Namespace namespace, - PolarisApplicationConfig configuration) - throws Exception { + PolarisApplicationConfig configuration) { MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java index 59842b6c8..4f6400864 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingCatalogTest.java @@ -28,6 +28,7 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -60,9 +61,12 @@ public class PolarisOverlappingCatalogTest { private static String realm; @BeforeAll - public static void setup(PolarisConnectionExtension.PolarisToken adminToken) { + public static void setup(PolarisConnectionExtension.PolarisToken adminToken) throws IOException { userToken = adminToken.token(); - realm = PolarisConnectionExtension.getTestRealm(PolarisServiceImplIntegrationTest.class); + realm = PolarisConnectionExtension.getTestRealm(PolarisOverlappingCatalogTest.class); + + // Set up the database location + PolarisConnectionExtension.createTestDir(realm); } private Response createCatalog(String prefix, String defaultBaseLocation, boolean isExternal) { diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java index 22a4c42b3..dce76c18a 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplIntegrationTest.java @@ -99,9 +99,12 @@ public class PolarisServiceImplIntegrationTest { private static String realm; @BeforeAll - public static void setup(PolarisConnectionExtension.PolarisToken adminToken) { + public static void setup(PolarisConnectionExtension.PolarisToken adminToken) throws IOException { userToken = adminToken.token(); realm = PolarisConnectionExtension.getTestRealm(PolarisServiceImplIntegrationTest.class); + + // Set up test location + PolarisConnectionExtension.createTestDir(realm); } @AfterEach diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java index b2ccb992c..ac1d5f2c0 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogIntegrationTest.java @@ -31,14 +31,11 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import org.apache.commons.io.FileUtils; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.Schema; @@ -133,9 +130,8 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests public static void setup() throws IOException { realm = PolarisConnectionExtension.getTestRealm(PolarisRestCatalogIntegrationTest.class); - Path testDir = Path.of("build/test_data/iceberg/" + realm); - FileUtils.deleteQuietly(testDir.toFile()); - Files.createDirectories(testDir); + // Set up test location + PolarisConnectionExtension.createTestDir(realm); } @BeforeEach diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java index 9b557bdda..b2b69a831 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisRestCatalogViewIntegrationTest.java @@ -29,11 +29,8 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.List; import java.util.Optional; -import org.apache.commons.io.FileUtils; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.rest.HTTPClient; @@ -96,9 +93,8 @@ public class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests "realmName"; + } @Test public void testCleanupFileNotExists() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -83,9 +89,6 @@ public void testCleanupFileNotExists() throws IOException { @Test public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -116,9 +119,6 @@ public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException @Test public void testCleanupFiles() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -166,9 +166,6 @@ public void close() { @Test public void testCleanupFilesWithRetries() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), diff --git a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java index d106a26e0..ab9f9324c 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/task/TableCleanupTaskHandlerTest.java @@ -41,17 +41,23 @@ import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.LoggerFactory; class TableCleanupTaskHandlerTest { + private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; + private RealmContext realmContext; + + @BeforeEach + void setUp() { + metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); + realmContext = () -> "realmName"; + } @Test public void testTableCleanup() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -112,9 +118,6 @@ public void testTableCleanup() throws IOException { @Test public void testTableCleanupHandlesAlreadyDeletedMetadata() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -174,9 +177,6 @@ public void close() { @Test public void testTableCleanupDuplicatesTasksIfFileStillExists() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -269,9 +269,6 @@ public void close() { @Test public void testTableCleanupMultipleSnapshots() throws IOException { - InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - RealmContext realmContext = () -> "realmName"; PolarisCallContext polarisCallContext = new PolarisCallContext( metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java index bc267c7eb..bc181a665 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java @@ -24,9 +24,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; +import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.polaris.core.context.CallContext; @@ -40,7 +45,9 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.auth.TokenUtils; import org.apache.polaris.service.config.PolarisApplicationConfig; +import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Namespace; @@ -50,7 +57,8 @@ import org.junit.platform.commons.util.ReflectionUtils; import org.slf4j.LoggerFactory; -public class PolarisConnectionExtension implements BeforeAllCallback, ParameterResolver { +public class PolarisConnectionExtension + implements BeforeAllCallback, AfterAllCallback, ParameterResolver { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private MetaStoreManagerFactory metaStoreManagerFactory; @@ -78,6 +86,9 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { PolarisApplicationConfig config = (PolarisApplicationConfig) dropwizardAppExtension.getConfiguration(); metaStoreManagerFactory = config.getMetaStoreManagerFactory(); + if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { + metaStoreManagerFactory.bootstrapRealms(List.of(realm)); + } RealmContext realmContext = config @@ -109,10 +120,40 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { } } + @Override + public void afterAll(ExtensionContext context) { + if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { + metaStoreManagerFactory.purgeRealms(List.of(realm)); + } + } + public static String getTestRealm(Class testClassName) { return testClassName.getName().replace('.', '_'); } + public static void createTestDir(String realm) throws IOException { + // Set up the database location + Path testDir = Path.of("build/test_data/polaris/" + realm); + if (Files.exists(testDir)) { + if (Files.isDirectory(testDir)) { + Files.walk(testDir) + .sorted(Comparator.reverseOrder()) + .forEach( + path -> { + try { + Files.delete(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + } else { + Files.delete(testDir); + } + } + Files.createDirectories(testDir); + } + static PolarisPrincipalSecrets getAdminSecrets() { return adminSecrets; } @@ -137,7 +178,11 @@ static PolarisPrincipalSecrets getAdminSecrets() { public boolean supportsParameter( ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - return parameterContext.getParameter().getType().equals(PolarisToken.class) + return parameterContext + .getParameter() + .getType() + .equals(PolarisConnectionExtension.PolarisToken.class) + || parameterContext.getParameter().getType().equals(MetaStoreManagerFactory.class) || parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class); } diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml index fd4a9d53e..ba8960669 100644 --- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml +++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml @@ -83,7 +83,7 @@ metaStoreManager: # type: remote # url: http://sdp-devvm-mcollado:8080 # type: eclipse-link # uncomment to use eclipse-link as metastore -# persistence-unit: polaris-dev +# persistence-unit: polaris io: factoryType: default From 1e4b50776eb33c8078c93bf78b4a32329c5ef778 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 12 Sep 2024 12:57:05 +0200 Subject: [PATCH 22/36] OpenApi generation: fix "reference not found" javadoc warnings (#286) --- server-templates/api.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-templates/api.mustache b/server-templates/api.mustache index 929dd6dda..0e8075282 100644 --- a/server-templates/api.mustache +++ b/server-templates/api.mustache @@ -90,7 +90,7 @@ public class {{classname}} { /** * {{^notes}}{{{summary}}}{{/notes}}{{{notes}}} * - * Response type {@link {{{returnBaseType}}}} + * Response type: {@code {{{returnBaseType}}}}. *{{#allParams}} @param {{paramName}} {{#required}}Required -{{/required}} {{description}} *{{/allParams}}{{#responses}} * @return {{{code}}} - {{{message}}}{{/responses}} From 6818d35dfd6b232b6be5ba6b002688b3ae4337a3 Mon Sep 17 00:00:00 2001 From: MonkeyCanCode Date: Thu, 12 Sep 2024 12:02:28 -0500 Subject: [PATCH 23/36] Fix circular dependency when ECLIPSELINK arg is set to true during build (#284) --- extension/persistence/eclipselink/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 3137a0cc0..50b7d41ef 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -24,7 +24,6 @@ plugins { dependencies { implementation(project(":polaris-core")) - runtimeOnly(project(":polaris-service")) implementation(libs.eclipselink) implementation(platform(libs.dropwizard.bom)) implementation("io.dropwizard:dropwizard-jackson") From 4c2a449e4e71eb60fdb0d9914d10c2f17e8f98d9 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 12 Sep 2024 23:20:52 +0530 Subject: [PATCH 24/36] Fix create view failure (#216) --- .../service/catalog/BasePolarisCatalog.java | 3 +++ .../catalog/PolarisSparkIntegrationTest.java | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index 2b1136e6f..5bd51be36 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -1164,6 +1164,9 @@ private class BasePolarisCatalogViewBuilder extends BaseMetastoreViewCatalog.Bas public BasePolarisCatalogViewBuilder(TableIdentifier identifier) { super(identifier); + withProperties( + PropertyUtil.propertiesWithPrefix( + BasePolarisCatalog.this.properties(), "table-default.")); } @Override diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java index 889a6580a..3160b0f52 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/PolarisSparkIntegrationTest.java @@ -240,6 +240,10 @@ private void cleanupCatalog(String catalogName) { for (Row table : tables) { onSpark("DROP TABLE " + namespace.getString(0) + "." + table.getString(1)); } + List views = onSpark("SHOW VIEWS IN " + namespace.getString(0)).collectAsList(); + for (Row view : views) { + onSpark("DROP VIEW " + namespace.getString(0) + "." + view.getString(1)); + } onSpark("DROP NAMESPACE " + namespace.getString(0)); } try (Response response = @@ -351,6 +355,20 @@ public void testCreateAndUpdateExternalTable() { assertThat(rowCount).isEqualTo(4); } + @Test + public void testCreateView() { + long namespaceCount = onSpark("SHOW NAMESPACES").count(); + assertThat(namespaceCount).isEqualTo(0L); + + onSpark("CREATE NAMESPACE ns1"); + onSpark("USE ns1"); + onSpark("CREATE TABLE tb1 (col1 integer, col2 string)"); + onSpark("INSERT INTO tb1 VALUES (1, 'a'), (2, 'b'), (3, 'c')"); + onSpark("CREATE VIEW view1 AS SELECT * FROM tb1"); + long recordCount = onSpark("SELECT * FROM view1").count(); + assertThat(recordCount).isEqualTo(3); + } + private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = EXT.client() From 19c82588199e69ed5fdddb9b42900c53d9889149 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Thu, 12 Sep 2024 13:55:43 -0400 Subject: [PATCH 25/36] Update spec to reflect that deleteCatalog is not cascading (#201) --- docs/index.html | 6 +++--- spec/index.yaml | 6 +----- spec/polaris-management-service.yml | 3 +-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/index.html b/docs/index.html index 861bce66e..df2c310b2 100644 --- a/docs/index.html +++ b/docs/index.html @@ -2220,8 +2220,8 @@

featureConfiguratio " class="sc-euGpHm sc-exayXG fwfkcU jYGAQp">

Server URL when the port can be inferred from the scheme

{scheme}://{host}/{basePath}/catalogs/{catalogName}

Generic base server URL, with all parts configurable

-
{scheme}://{host}:{port}/{basePath}/catalogs/{catalogName}

Request samples

Content type
application/json
{
  • "currentEntityVersion": 0,
  • "properties": {
    },
  • "storageConfigInfo": {
    }
}

Response samples

Content type
application/json
Example
{
  • "type": "INTERNAL",
  • "name": "string",
  • "properties": {
    },
  • "createTimestamp": 0,
  • "lastUpdateTimestamp": 0,
  • "entityVersion": 0,
  • "storageConfigInfo": {
    }
}

deleteCatalog

Delete an existing catalog. This is a cascading operation that deletes all metadata, including principals, roles and grants. If the catalog is an internal catalog, all tables and namespaces are dropped without purge.

+
{scheme}://{host}:{port}/{basePath}/catalogs/{catalogName}

Request samples

Content type
application/json
{
  • "currentEntityVersion": 0,
  • "properties": {
    },
  • "storageConfigInfo": {
    }
}

Response samples

Content type
application/json
Example
{
  • "type": "INTERNAL",
  • "name": "string",
  • "properties": {
    },
  • "createTimestamp": 0,
  • "lastUpdateTimestamp": 0,
  • "entityVersion": 0,
  • "storageConfigInfo": {
    }
}

deleteCatalog

Delete an existing catalog. The catalog must be empty.

Authorizations:
Polaris_Management_Service_OAuth2
path Parameters
catalogName
required
string [ 1 .. 256 ] characters ^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$

The name of the catalog

Responses

Request samples

Content type
application/json
{
  • "source": {
    },
  • "destination": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}