[Read] Implementation of Hash Bucketing and Batch Processing

.asf.yaml

@@ -39,6 +39,8 @@ github:
         required_approving_review_count: 1
 
       required_linear_history: true
+
+  del_branch_on_merge: true
 
   features:
     wiki: true

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,9 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "1.6.1 (latest release)"
+        - "1.7.1 (latest release)"
+        - "1.7.0"
+        - "1.6.1"
         - "1.6.0"
         - "1.5.2"
         - "1.5.1"

.github/labeler.yml

@@ -130,12 +130,6 @@ MR:
       'mr/**/*'
     ]
 
-PIG:
-  - changed-files:
-    - any-glob-to-any-file: [
-      'pig/**/*'
-    ]
-
 AWS:
   - changed-files:
     - any-glob-to-any-file: [

.github/workflows/delta-conversion-ci.yml

@@ -37,23 +37,22 @@ on:
       - '.github/workflows/jmh-benchmarks-ci.yml'
       - '.github/workflows/kafka-connect-ci.yml'
       - '.github/workflows/labeler.yml'
-      - '.github/workflows/licence-check.yml'
+      - '.github/workflows/license-check.yml'
       - '.github/workflows/open-api.yml'
       - '.github/workflows/publish-snapshot.yml'
       - '.github/workflows/recurring-jmh-benchmarks.yml'
       - '.github/workflows/site-ci.yml'
       - '.github/workflows/spark-ci.yml'
       - '.github/workflows/stale.yml'
       - '.gitignore'
-      - '.asf.yml'
+      - '.asf.yaml'
       - 'dev/**'
       - 'mr/**'
       - 'hive3/**'
       - 'hive3-orc-bundle/**'
       - 'hive-runtime/**'
       - 'flink/**'
       - 'kafka-connect/**'
-      - 'pig/**'
       - 'docs/**'
       - 'site/**'
       - 'open-api/**'

.github/workflows/flink-ci.yml

@@ -37,23 +37,22 @@ on:
     - '.github/workflows/jmh-benchmarks-ci.yml'
     - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
-    - '.github/workflows/licence-check.yml'
+    - '.github/workflows/license-check.yml'
     - '.github/workflows/open-api.yml'
     - '.github/workflows/publish-snapshot.yml'
     - '.github/workflows/recurring-jmh-benchmarks.yml'
     - '.github/workflows/site-ci.yml'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/stale.yml'
     - '.gitignore'
-    - '.asf.yml'
+    - '.asf.yaml'
     - 'dev/**'
     - 'mr/**'
     - 'hive3/**'
     - 'hive3-orc-bundle/**'
     - 'hive-runtime/**'
     - 'kafka-connect/**'
     - 'spark/**'
-    - 'pig/**'
     - 'docs/**'
     - 'site/**'
     - 'open-api/**'

.github/workflows/hive-ci.yml

@@ -37,21 +37,20 @@ on:
     - '.github/workflows/jmh-benchmarks-ci.yml'
     - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
-    - '.github/workflows/licence-check.yml'
+    - '.github/workflows/license-check.yml'
     - '.github/workflows/open-api.yml'
     - '.github/workflows/publish-snapshot.yml'
     - '.github/workflows/recurring-jmh-benchmarks.yml'
     - '.github/workflows/site-ci.yml'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/stale.yml'
     - '.gitignore'
-    - '.asf.yml'
+    - '.asf.yaml'
     - 'dev/**'
     - 'arrow/**'
     - 'spark/**'
     - 'flink/**'
     - 'kafka-connect/**'
-    - 'pig/**'
     - 'docs/**'
     - 'site/**'
     - 'open-api/**'

.github/workflows/java-ci.yml

@@ -37,15 +37,15 @@ on:
     - '.github/workflows/jmh-benchmarks-ci.yml'
     - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
-    - '.github/workflows/licence-check.yml'
+    - '.github/workflows/license-check.yml'
     - '.github/workflows/open-api.yml'
     - '.github/workflows/publish-snapshot.yml'
     - '.github/workflows/recurring-jmh-benchmarks.yml'
     - '.github/workflows/site-ci.yml'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/stale.yml'
     - '.gitignore'
-    - '.asf.yml'
+    - '.asf.yaml'
     - 'dev/**'
     - 'docs/**'
     - 'site/**'

.github/workflows/kafka-connect-ci.yml

@@ -37,23 +37,22 @@ on:
     - '.github/workflows/java-ci.yml'
     - '.github/workflows/jmh-benchmarks-ci.yml'
     - '.github/workflows/labeler.yml'
-    - '.github/workflows/licence-check.yml'
+    - '.github/workflows/license-check.yml'
     - '.github/workflows/open-api.yml'
     - '.github/workflows/publish-snapshot.yml'
     - '.github/workflows/recurring-jmh-benchmarks.yml'
     - '.github/workflows/site-ci.yml'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/stale.yml'
     - '.gitignore'
-    - '.asf.yml'
+    - '.asf.yaml'
     - 'dev/**'
     - 'mr/**'
     - 'flink/**'
     - 'hive3/**'
     - 'hive3-orc-bundle/**'
     - 'hive-runtime/**'
     - 'spark/**'
-    - 'pig/**'
     - 'docs/**'
     - 'site/**'
     - 'open-api/**'

.github/workflows/publish-iceberg-rest-fixture-docker.yml

@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Build and Push 'iceberg-rest-fixture' Docker Image
+
+on:
+  push:
+    tags:
+      - 'apache-iceberg-[0-9]+.[0-9]+.[0-9]+'
+  workflow_dispatch:
+
+env:
+  DOCKER_IMAGE_TAG: iceberg-rest-fixture
+  DOCKER_IMAGE_VERSION: latest
+  DOCKER_REPOSITORY: apache
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-java@v4
+      with:
+        distribution: zulu
+        java-version: 21
+    - name: Build Iceberg Open API project
+      run: ./gradlew :iceberg-open-api:shadowJar
+    - name: Login to Docker Hub
+      run: |
+        docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Set the tagged version
+      # for tag 'apache-iceberg-1.7.1', publish image 'apache/iceberg-rest-fixture:1.7.1'
+      if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+      run: |
+        echo "DOCKER_IMAGE_VERSION=`echo ${{ github.ref }} | tr -d -c 0-9.`" >> "$GITHUB_ENV"
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Build and Push
+      uses: docker/build-push-action@v6
+      with:
+        context: ./
+        file: ./docker/iceberg-rest-fixture/Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: ${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_IMAGE_TAG }}:${{ env.DOCKER_IMAGE_VERSION }}

.github/workflows/spark-ci.yml

@@ -38,14 +38,14 @@ on:
     - '.github/workflows/jmh-benchmarks-ci.yml'
     - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
-    - '.github/workflows/licence-check.yml'
+    - '.github/workflows/license-check.yml'
     - '.github/workflows/open-api.yml'
     - '.github/workflows/publish-snapshot.yml'
     - '.github/workflows/recurring-jmh-benchmarks.yml'
     - '.github/workflows/site-ci.yml'
     - '.github/workflows/stale.yml'
     - '.gitignore'
-    - '.asf.yml'
+    - '.asf.yaml'
     - 'dev/**'
     - 'site/**'
     - 'mr/**'
@@ -54,7 +54,6 @@ on:
     - 'hive-runtime/**'
     - 'flink/**'
     - 'kafka-connect/**'
-    - 'pig/**'
     - 'docs/**'
     - 'open-api/**'
     - 'format/**'

.palantir/revapi.yml

@@ -1145,6 +1145,32 @@ acceptedBreaks:
       new: "method org.apache.iceberg.BaseMetastoreOperations.CommitStatus org.apache.iceberg.BaseMetastoreTableOperations::checkCommitStatus(java.lang.String,\
         \ org.apache.iceberg.TableMetadata)"
       justification: "Removing deprecated code"
+  "1.7.0":
+    org.apache.iceberg:iceberg-core:
+    - code: "java.method.removed"
+      old: "method <T extends org.apache.iceberg.StructLike> org.apache.iceberg.deletes.PositionDeleteIndex\
+        \ org.apache.iceberg.deletes.Deletes::toPositionIndex(java.lang.CharSequence,\
+        \ java.util.List<org.apache.iceberg.io.CloseableIterable<T>>)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method <T extends org.apache.iceberg.StructLike> org.apache.iceberg.deletes.PositionDeleteIndex\
+        \ org.apache.iceberg.deletes.Deletes::toPositionIndex(java.lang.CharSequence,\
+        \ java.util.List<org.apache.iceberg.io.CloseableIterable<T>>, java.util.concurrent.ExecutorService)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method <T> org.apache.iceberg.io.CloseableIterable<T> org.apache.iceberg.deletes.Deletes::streamingFilter(org.apache.iceberg.io.CloseableIterable<T>,\
+        \ java.util.function.Function<T, java.lang.Long>, org.apache.iceberg.io.CloseableIterable<java.lang.Long>)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method <T> org.apache.iceberg.io.CloseableIterable<T> org.apache.iceberg.deletes.Deletes::streamingFilter(org.apache.iceberg.io.CloseableIterable<T>,\
+        \ java.util.function.Function<T, java.lang.Long>, org.apache.iceberg.io.CloseableIterable<java.lang.Long>,\
+        \ org.apache.iceberg.deletes.DeleteCounter)"
+      justification: "Removing deprecated code"
+    - code: "java.method.removed"
+      old: "method <T> org.apache.iceberg.io.CloseableIterable<T> org.apache.iceberg.deletes.Deletes::streamingMarker(org.apache.iceberg.io.CloseableIterable<T>,\
+        \ java.util.function.Function<T, java.lang.Long>, org.apache.iceberg.io.CloseableIterable<java.lang.Long>,\
+        \ java.util.function.Consumer<T>)"
+      justification: "Removing deprecated code"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

LICENSE

@@ -298,6 +298,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 This product includes code from Delta Lake.
 
 * AssignmentAlignmentSupport is an independent development but UpdateExpressionsSupport in Delta was used as a reference.
+* RoaringPositionBitmap is a Java implementation of RoaringBitmapArray in Delta.
 
 Copyright: 2020 The Delta Lake Project Authors.
 Home page: https://delta.io/

README.md

@@ -74,7 +74,6 @@ Iceberg also has modules for adding Iceberg support to processing engines:
 * `iceberg-spark` is an implementation of Spark's Datasource V2 API for Iceberg with submodules for each spark versions (use runtime jars for a shaded version)
 * `iceberg-flink` contains classes for integrating with Apache Flink (use iceberg-flink-runtime for a shaded version)
 * `iceberg-mr` contains an InputFormat and other classes for integrating with Apache Hive
-* `iceberg-pig` is an implementation of Pig's LoadFunc API for Iceberg
 
 ---
 **NOTE**
@@ -98,3 +97,4 @@ This repository contains the Java implementation of Iceberg. Other implementatio
 * **Go**: [iceberg-go](https://github.com/apache/iceberg-go)
 * **PyIceberg** (Python): [iceberg-python](https://github.com/apache/iceberg-python)
 * **Rust**: [iceberg-rust](https://github.com/apache/iceberg-rust)
+* **C++**: [iceberg-cpp](https://github.com/apache/iceberg-cpp)

aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java

@@ -65,7 +65,7 @@ public static AliyunOSSExtension initialize() {
     } else {
       LOG.info(
           "Initializing AliyunOSSExtension implementation with default AliyunOSSMockExtension");
-      extension = AliyunOSSMockExtension.builder().silent().build();
+      extension = AliyunOSSMockExtension.builder().build();
     }
 
     return extension;

aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSOutputStream.java

@@ -70,7 +70,7 @@ public void testWrite() throws IOException {
       reset(ossMock);
 
       // Write large file.
-      writeAndVerify(ossMock, uri, randomData(32 * 1024 * 1024), arrayWrite);
+      writeAndVerify(ossMock, uri, randomData(32 * 1024), arrayWrite);
       verify(ossMock, times(1)).putObject(any());
       reset(ossMock);
     }