Remove Hive 2 dependency

apache · Aug 26, 2024 · f55a620 · f55a620
1 parent cbd71eb
commit f55a620
Show file tree

Hide file tree

Showing 28 changed files with 71 additions and 563 deletions.
diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml
@@ -65,35 +65,6 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  hive2-tests:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        jvm: [11, 17, 21]
-    env:
-      SPARK_LOCAL_IP: localhost
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-java@v4
-      with:
-        distribution: zulu
-        java-version: ${{ matrix.jvm }}
-    - uses: actions/cache@v4
-      with:
-        path: |
-          ~/.gradle/caches
-          ~/.gradle/wrapper
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-        restore-keys: ${{ runner.os }}-gradle-
-    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
-    - uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-        name: test logs
-        path: |
-          **/build/testlogs
-
   hive3-tests:
     runs-on: ubuntu-22.04
     strategy:

diff --git a/build.gradle b/build.gradle
@@ -681,7 +681,7 @@ project(':iceberg-hive-metastore') {
 
     compileOnly libs.avro.avro
 
-    compileOnly(libs.hive2.metastore) {
+    compileOnly(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -701,7 +701,7 @@ project(':iceberg-hive-metastore') {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -713,7 +713,7 @@ project(':iceberg-hive-metastore') {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -729,7 +729,9 @@ project(':iceberg-hive-metastore') {
       exclude group: 'com.zaxxer', module: 'HikariCP'
     }
 
-    compileOnly(libs.hadoop2.client) {
+    testImplementation(libs.hive3.standalone.metastore)
+
+    compileOnly(libs.hadoop3.client) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
     }

diff --git a/flink/v1.18/build.gradle b/flink/v1.18/build.gradle
@@ -88,7 +88,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -100,7 +100,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -192,7 +192,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    integrationImplementation(libs.hive2.metastore) {
+    integrationImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -209,7 +209,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.slf4j'
     }
 
-    integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    integrationImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

diff --git a/flink/v1.19/build.gradle b/flink/v1.19/build.gradle
@@ -88,7 +88,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -100,7 +100,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -192,7 +192,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    integrationImplementation(libs.hive2.metastore) {
+    integrationImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -209,7 +209,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.slf4j'
     }
 
-    integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    integrationImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

diff --git a/flink/v1.20/build.gradle b/flink/v1.20/build.gradle
@@ -88,7 +88,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -100,7 +100,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -192,7 +192,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    integrationImplementation(libs.hive2.metastore) {
+    integrationImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -209,7 +209,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.slf4j'
     }
 
-    integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    integrationImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

diff --git a/gradle.properties b/gradle.properties
@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=1.20
 systemProp.knownFlinkVersions=1.18,1.19,1.20
-systemProp.defaultHiveVersions=2
-systemProp.knownHiveVersions=2,3
+systemProp.defaultHiveVersions=3
+systemProp.knownHiveVersions=3
 systemProp.defaultSparkVersions=3.5
 systemProp.knownSparkVersions=3.3,3.4,3.5
 systemProp.defaultScalaVersion=2.12

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -47,7 +47,6 @@ guava = "33.3.0-jre"
 hadoop2 = "2.7.3"
 hadoop3 = "3.3.6"
 httpcomponents-httpclient5 = "5.3.1"
-hive2 = { strictly = "2.3.9"} # see rich version usage explanation above
 hive3 = "3.1.3"
 immutables-value = "2.10.1"
 jackson-bom = "2.14.2"
@@ -135,12 +134,9 @@ hadoop2-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-c
 hadoop2-minicluster = { module = "org.apache.hadoop:hadoop-minicluster", version.ref = "hadoop2" }
 hadoop3-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop3" }
 hadoop3-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop3" }
-hive2-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive2" }
-hive2-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive2" }
-hive2-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive2" }
-hive2-service = { module = "org.apache.hive:hive-service", version.ref = "hive2" }
 hive3-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive3" }
 hive3-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive3" }
+hive3-standalone-metastore = { module = "org.apache.hive:hive-standalone-metastore", version.ref = "hive3" }
 hive3-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive3" }
 hive3-service = { module = "org.apache.hive:hive-service", version.ref = "hive3" }
 httpcomponents-httpclient5 = { module = "org.apache.httpcomponents.client5:httpclient5", version.ref = "httpcomponents-httpclient5" }

diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java
@@ -509,7 +509,7 @@ public void testRegisterHadoopTableToHiveCatalog() throws IOException, TExceptio
 
     assertThatThrownBy(() -> HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, "table1"))
         .isInstanceOf(NoSuchObjectException.class)
-        .hasMessage("hivedb.table1 table not found");
+        .hasMessage("hive.hivedb.table1 table not found");
     assertThatThrownBy(() -> catalog.loadTable(identifier))
         .isInstanceOf(NoSuchTableException.class)
         .hasMessage("Table does not exist: hivedb.table1");

diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java
@@ -30,12 +30,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Function;
 import org.apache.hadoop.hive.metastore.api.FunctionType;
 import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.thrift.transport.TTransportException;
 import org.junit.jupiter.api.AfterEach;
@@ -121,36 +121,36 @@ public void testGetTablesFailsForNonReconnectableException() throws Exception {
 
   @Test
   public void testExceptionMessages() {
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException(new MetaException("Another meta exception")));
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)
           .hasMessage("Failed to connect to Hive Metastore");
     }
 
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException(new MetaException()));
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)
           .hasMessage("Failed to connect to Hive Metastore");
     }
 
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException());
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)
           .hasMessage("Failed to connect to Hive Metastore");
     }
 
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException("Another instance of Derby may have already booted"));
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)

diff --git a/hive3/build.gradle b/hive3/build.gradle
@@ -27,13 +27,6 @@ project(':iceberg-hive3') {
     }
   }
 
-  // exclude these Hive2-specific tests from iceberg-mr
-  test {
-    exclude '**/TestIcebergDateObjectInspector.class'
-    exclude '**/TestIcebergTimestampObjectInspector.class'
-    exclude '**/TestIcebergTimestampWithZoneObjectInspector.class'
-  }
-
   dependencies {
     compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
     compileOnly project(':iceberg-api')

diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle
@@ -134,7 +134,7 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {
     implementation 'com.azure:azure-identity'
 
     hive project(':iceberg-hive-metastore')
-    hive(libs.hive2.metastore) {
+    hive(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

diff --git a/mr/build.gradle b/mr/build.gradle
@@ -37,11 +37,11 @@ project(':iceberg-mr') {
     implementation project(':iceberg-orc')
     implementation project(':iceberg-parquet')
 
-    compileOnly(libs.hadoop2.client) {
+    compileOnly(libs.hadoop3.client) {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    compileOnly("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    compileOnly("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
       exclude group: 'com.google.guava'
       exclude group: 'com.google.protobuf', module: 'protobuf-java'
@@ -52,8 +52,8 @@ project(':iceberg-mr') {
       exclude group: 'org.pentaho' // missing dependency
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
     }
-    compileOnly libs.hive2.metastore
-    compileOnly libs.hive2.serde
+    compileOnly libs.hive3.metastore
+    compileOnly libs.hive3.serde
 
     implementation libs.caffeine
 
@@ -70,7 +70,7 @@ project(':iceberg-mr') {
     testImplementation libs.kryo.shaded
     testImplementation platform(libs.jackson.bom)
     testImplementation libs.jackson.annotations
-    testImplementation(libs.hive2.service) {
+    testImplementation(libs.hive3.service) {
       exclude group: 'org.apache.hive', module: 'hive-exec'
     }
     testImplementation libs.tez08.dag

diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
@@ -134,9 +134,7 @@ public boolean shouldSkipCombine(Path path, Configuration conf) {
     return true;
   }
 
-  // Override annotation commented out, since this interface method has been introduced only in Hive
-  // 3
-  // @Override
+  @Override
   public VectorizedSupport.Support[] getSupportedFeatures() {
     return new VectorizedSupport.Support[0];
   }

diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -106,9 +106,7 @@ public void configureOutputJobProperties(TableDesc tableDesc, Map<String, String
   @Override
   public void configureTableJobProperties(TableDesc tableDesc, Map<String, String> map) {}
 
-  // Override annotation commented out, since this interface method has been introduced only in Hive
-  // 3
-  // @Override
+  @Override
   public void configureInputJobCredentials(TableDesc tableDesc, Map<String, String> secrets) {}
 
   @Override