Merge branch 'main' into feature/OTF-1500_column_comparison

* main: (208 commits) Docs: Fix Flink 1.20 support versions (apache#11065) Flink: Fix compile warning (apache#11072) Docs: Initial committer guidelines and requirements for merging (apache#10780) Core: Refactor ZOrderByteUtils (apache#10624) API: implement types timestamp_ns and timestamptz_ns (apache#9008) Build: Bump com.google.errorprone:error_prone_annotations (apache#11055) Build: Bump mkdocs-material from 9.5.33 to 9.5.34 (apache#11062) Flink: Backport PR apache#10526 to v1.18 and v1.20 (apache#11018) Kafka Connect: Disable publish tasks in runtime project (apache#11032) Flink: add unit tests for range distribution on bucket partition column (apache#11033) Spark 3.5: Use FileGenerationUtil in PlanningBenchmark (apache#11027) Core: Add benchmark for appending files (apache#11029) Build: Ignore benchmark output folders across all modules (apache#11030) Spec: Add RemovePartitionSpecsUpdate REST update type (apache#10846) Docs: bump latest version to 1.6.1 (apache#11036) OpenAPI, Build: Apply spotless to testFixtures source code (apache#11024) Core: Generate realistic bounds in benchmarks (apache#11022) Add REST Compatibility Kit (apache#10908) Flink: backport PR apache#10832 of inferring parallelism in FLIP-27 source (apache#11009) Docs: Add Druid docs url to sidebar (apache#10997) ...
Teradata · Sep 17, 2024 · 7e275b0 · 7e275b0
2 parents 9f2cb8e + 7830a3b
commit 7e275b0
Show file tree

Hide file tree

Showing 1,011 changed files with 53,114 additions and 11,666 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -49,12 +49,12 @@ github:
     - jun-he
     - marton-bod
     - samarthjain
-    - findepi
     - SreeramGarlapati
     - samredai
     - gaborkaszab
     - bitsondatadev
     - ajantha-bhat
+    - jbonofre
   ghp_branch: gh-pages
   ghp_path: /
 

diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml
@@ -159,7 +159,6 @@
         </module>
         <module name="ClassTypeParameterName"> <!-- Java Style Guide: Type variable names -->
             <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
-            <message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="CovariantEquals"/> <!-- Java Coding Guidelines: Override ``Object#equals`` consistently -->
         <module name="DefaultComesLast"/> <!-- Java Style Guide: The default case is present -->
@@ -281,12 +280,13 @@
         </module>
         <module name="InnerAssignment"/> <!-- Java Coding Guidelines: Inner assignments: Not used -->
         <module name="MemberName"> <!-- Java Style Guide: Non-constant field names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
-            <message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z][a-zA-Z0-9]++$"/>
+        </module>
+        <module name="ConstantName">
+            <property name="format" value="^[A-Z][A-Z0-9]*+(_[A-Z0-9]++)*+$"/>
         </module>
         <module name="MethodName"> <!-- Java Style Guide: Method names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9_]+$"/>
-            <message key="name.invalidPattern" value="Method name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z][a-zA-Z0-9_]++$"/>
         </module>
         <module name="MethodParamPad"/> <!-- Java Style Guide: Horizontal whitespace -->
         <module name="MissingDeprecated"/> <!-- Java Coding Guide: Deprecate per annotation and Javadoc -->
@@ -304,8 +304,7 @@
         <module name="PackageAnnotation"/> <!-- Java Style Guide: Package statement -->
         <module name="PackageDeclaration"/> <!-- Java Style Guide: Package statement -->
         <module name="PackageName"> <!-- Java Style Guide: Package names -->
-            <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
-            <message key="name.invalidPattern" value="Package name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z]++(\.[a-z][a-z0-9]*+)*+$"/>
         </module>
         <module name="ParameterAssignment"/> <!-- Java Coding Guidelines: Final variables and parameters -->
         <module name="ParenPad"/> <!-- Java Style Guide: Horizontal whitespace -->
@@ -415,6 +414,9 @@
             <property name="format" value="@Test\(.*expected.*\)"/>
             <property name="message" value="Prefer using Assertions.assertThatThrownBy(...).isInstanceOf(...) instead."/>
         </module>
+        <module name="IllegalToken">
+            <property name="tokens" value="LITERAL_ASSERT"/>
+        </module>
         <module name="IllegalImport">
             <property name="id" value="BanExpectedExceptionUsage"/>
             <property name="illegalClasses" value="org.junit.rules.ExpectedException"/>
@@ -444,7 +446,6 @@
             <message key="todo.match" value="There must be whitespace at the beginning of all comments."/>
         </module>
         <module name="TypeName"> <!-- Java Style Guide: Class names -->
-            <message key="name.invalidPattern" value="Type name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="TypecastParenPad"/> <!-- Java Style Guide: Horizontal whitespace -->
         <module name="UnusedImports"> <!-- Java Style Guide: No unused imports -->
@@ -480,23 +481,20 @@
         <module name="LocalFinalVariableName"/> <!-- Java Style Guide: Local variable names -->
         <module name="LocalVariableName"> <!-- Java Style Guide: Local variable names -->
             <property name="tokens" value="VARIABLE_DEF"/>
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
+            <property name="format" value="^[a-z][a-zA-Z0-9]++$"/>
             <property name="allowOneCharVarInForLoop" value="true"/>
-            <message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="MethodLength"/> <!-- Java Coding Guide: Methods and functions: focused, crisp, concise -->
         <module name="MethodTypeParameterName"> <!-- Java Style Guide: Type variable names -->
             <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
-            <message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="NestedForDepth">
             <property name="max" value="2"/>
         </module>
         <module name="NestedTryDepth"/> <!-- Java Coding Guide: Try/catch blocks: never nested -->
         <module name="NonEmptyAtclauseDescription"/> <!-- Java Style Guide: At-clauses -->
         <module name="ParameterName"> <!-- Java Style Guide: Parameter names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
-            <message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z][a-zA-Z0-9]++$"/>
             <property name="ignoreOverridden" value="true"/>
         </module>
 

diff --git a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml
@@ -28,7 +28,9 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "1.5.2 (latest release)"
+        - "1.6.1 (latest release)"
+        - "1.6.0"
+        - "1.5.2"
         - "1.5.1"
         - "1.5.0"
         - "1.4.3"
@@ -76,3 +78,11 @@ body:
         You can include files by dragging and dropping them here.
     validations:
       required: true
+  - type: checkboxes
+    attributes:
+      label: Willingness to contribute
+      description: The Apache Iceberg community encourages bug fix contributions. Would you or another member of your organization be willing to contribute a fix for this bug to the Apache Iceberg codebase?
+      options:
+        - label: I can contribute a fix for this bug independently
+        - label: I would be willing to contribute a fix for this bug with guidance from the Iceberg community
+        - label: I cannot contribute a fix for this bug at this time
diff --git a/.github/ISSUE_TEMPLATE/iceberg_improvement.yml b/.github/ISSUE_TEMPLATE/iceberg_improvement.yml
@@ -50,4 +50,12 @@ body:
         - Hive
         - Other
     validations:
-      required: false
+      required: false
+  - type: checkboxes
+    attributes:
+      label: Willingness to contribute
+      description: The Apache Iceberg community encourages contributions. Would you or another member of your organization be willing to contribute this improvement/feature to the Apache Iceberg codebase?
+      options:
+        - label: I can contribute this improvement/feature independently
+        - label: I would be willing to contribute this improvement/feature with guidance from the Iceberg community
+        - label: I cannot contribute this improvement/feature at this time
diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -100,7 +100,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:

diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml
@@ -73,12 +73,8 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
-        flink: ['1.17', '1.18', '1.19']
-        exclude:
-          # Flink 1.17 does not support Java 17.
-          - jvm: 17
-            flink: '1.17'
+        jvm: [11, 17, 21]
+        flink: ['1.18', '1.19', '1.20']
     env:
       SPARK_LOCAL_IP: localhost
     steps:

diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml
@@ -69,7 +69,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -98,7 +98,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:

diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml
@@ -65,7 +65,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
@@ -107,7 +107,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4

diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
@@ -37,7 +37,7 @@ jobs:
       - uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: 8
+          java-version: 11
       - run: |
           ./gradlew printVersion
           ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}

diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml
@@ -71,9 +71,16 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
         spark: ['3.3', '3.4', '3.5']
         scala: ['2.12', '2.13']
+        exclude:
+          # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
+          # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
+          - jvm: 21
+            spark: '3.3'
+          - jvm: 21
+            spark: '3.4'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
@@ -25,6 +25,7 @@ on:
 permissions:
   # All other permissions are set to none
   issues: write
+  pull-requests: write
 
 jobs:
   stale:
@@ -54,4 +55,4 @@ jobs:
           days-before-pr-stale: 30
           days-before-pr-close: 7
           ascending: true
-          operations-per-run: 100
+          operations-per-run: 200
diff --git a/.gitignore b/.gitignore
@@ -37,7 +37,7 @@ spark/v3.4/spark/benchmark/*
 spark/v3.4/spark-extensions/benchmark/*
 spark/v3.5/spark/benchmark/*
 spark/v3.5/spark-extensions/benchmark/*
-data/benchmark/*
+*/benchmark/*
 
 __pycache__/
 *.py[cod]

diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml
@@ -1025,14 +1025,11 @@ acceptedBreaks:
       new: "class org.apache.iceberg.types.Types.NestedField"
       justification: "new Constructor added"
     org.apache.iceberg:iceberg-core:
-    - code: "java.method.visibilityReduced"
-      old: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
-      new: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
-      justification: "Deprecations for 1.6.0 release"
     - code: "java.element.noLongerDeprecated"
       old: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
       new: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
-      justification: "Constructor became private as part of deprecations cleanup for 1.6.0 release"
+      justification: "Constructor became private as part of deprecations cleanup for\
+        \ 1.6.0 release"
     - code: "java.element.noLongerDeprecated"
       old: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
         \ java.lang.String>, java.lang.String, java.lang.String, java.lang.String,\
@@ -1056,6 +1053,89 @@ acceptedBreaks:
     - code: "java.method.removed"
       old: "method org.apache.iceberg.DataFiles.Builder org.apache.iceberg.DataFiles.Builder::withEqualityFieldIds(java.util.List<java.lang.Integer>)"
       justification: "Deprecations for 1.6.0 release"
+    - code: "java.method.visibilityReduced"
+      old: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
+      new: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
+      justification: "Deprecations for 1.6.0 release"
+  "1.6.0":
+    org.apache.iceberg:iceberg-common:
+    - code: "java.method.removed"
+      old: "method <T> org.apache.iceberg.common.DynFields.StaticField<T> org.apache.iceberg.common.DynFields.Builder::buildStaticChecked()\
+        \ throws java.lang.NoSuchFieldException"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method java.lang.Class<? extends C> org.apache.iceberg.common.DynConstructors.Ctor<C>::getConstructedClass()"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.common.DynConstructors.Builder org.apache.iceberg.common.DynConstructors.Builder::hiddenImpl(java.lang.Class<?>[])"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.common.DynMethods.Builder org.apache.iceberg.common.DynMethods.Builder::ctorImpl(java.lang.Class<?>,\
+        \ java.lang.Class<?>[])"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.common.DynMethods.Builder org.apache.iceberg.common.DynMethods.Builder::ctorImpl(java.lang.String,\
+        \ java.lang.Class<?>[])"
+      justification: "Removing deprecated code"
+    - code: "java.method.visibilityReduced"
+      old: "method <R> R org.apache.iceberg.common.DynMethods.UnboundMethod::invokeChecked(java.lang.Object,\
+        \ java.lang.Object[]) throws java.lang.Exception"
+      new: "method <R> R org.apache.iceberg.common.DynMethods.UnboundMethod::invokeChecked(java.lang.Object,\
+        \ java.lang.Object[]) throws java.lang.Exception"
+      justification: "Reduced visibility and scoped to package"
+    org.apache.iceberg:iceberg-core:
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.GenericManifestFile"
+      new: "class org.apache.iceberg.GenericManifestFile"
+      justification: "Serialization across versions is not supported"
+    - code: "java.class.removed"
+      old: "enum org.apache.iceberg.BaseMetastoreTableOperations.CommitStatus"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method java.lang.String org.apache.iceberg.FileScanTaskParser::toJson(org.apache.iceberg.FileScanTask)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.FileScanTask org.apache.iceberg.FileScanTaskParser::fromJson(java.lang.String,\
+        \ boolean)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.ContentCache.CacheEntry org.apache.iceberg.io.ContentCache::get(java.lang.String,\
+        \ java.util.function.Function<java.lang.String, org.apache.iceberg.io.ContentCache.FileContent>)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.ContentCache.CacheEntry org.apache.iceberg.io.ContentCache::getIfPresent(java.lang.String)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.InputFile org.apache.iceberg.io.ContentCache::tryCache(org.apache.iceberg.io.FileIO,\
+        \ java.lang.String, long)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.BaseOverwriteFiles"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.BaseReplacePartitions"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.BaseRewriteManifests"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.io.OutputFile org.apache.iceberg.SnapshotProducer<ThisT>::newManifestOutput()\
+        \ @ org.apache.iceberg.StreamingDelete"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
+        \ java.lang.String>, java.lang.String, java.lang.String, java.lang.String,\
+        \ java.lang.String, java.lang.String)"
+      justification: "Removing deprecated code"
+    - code: "java.method.returnTypeChanged"
+      old: "method org.apache.iceberg.BaseMetastoreTableOperations.CommitStatus org.apache.iceberg.BaseMetastoreTableOperations::checkCommitStatus(java.lang.String,\
+        \ org.apache.iceberg.TableMetadata)"
+      new: "method org.apache.iceberg.BaseMetastoreOperations.CommitStatus org.apache.iceberg.BaseMetastoreTableOperations::checkCommitStatus(java.lang.String,\
+        \ org.apache.iceberg.TableMetadata)"
+      justification: "Removing deprecated code"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

diff --git a/README.md b/README.md
@@ -51,7 +51,7 @@ Community discussions happen primarily on the [dev mailing list][dev-list] or on
 
 ### Building
 
-Iceberg is built using Gradle with Java 8, 11, or 17.
+Iceberg is built using Gradle with Java 11, 17, or 21.
 
 * To invoke a build and run tests: `./gradlew build`
 * To skip tests: `./gradlew build -x test -x integrationTest`

diff --git a/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSInputStream.java b/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSInputStream.java
@@ -157,7 +157,7 @@ private void closeStream() throws IOException {
     }
   }
 
-  @SuppressWarnings("checkstyle:NoFinalizer")
+  @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize"})
   @Override
   protected void finalize() throws Throwable {
     super.finalize();

diff --git a/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSOutputStream.java b/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSOutputStream.java
@@ -165,7 +165,7 @@ private void cleanUpStagingFiles() {
     }
   }
 
-  @SuppressWarnings("checkstyle:NoFinalizer")
+  @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize"})
   @Override
   protected void finalize() throws Throwable {
     super.finalize();