Hive: Drop Hive 2 dependency

.github/workflows/hive-ci.yml

@@ -66,7 +66,7 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  hive2-tests:
+  mr-tests:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
@@ -87,7 +87,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: ${{ runner.os }}-gradle-
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -DkafkaVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -DkafkaVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
     - uses: actions/upload-artifact@v4
       if: failure()
       with:

build.gradle

@@ -675,7 +675,7 @@ project(':iceberg-hive-metastore') {
 
     compileOnly libs.avro.avro
 
-    compileOnly(libs.hive2.metastore) {
+    compileOnly(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -695,7 +695,7 @@ project(':iceberg-hive-metastore') {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -707,7 +707,7 @@ project(':iceberg-hive-metastore') {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -723,7 +723,9 @@ project(':iceberg-hive-metastore') {
       exclude group: 'com.zaxxer', module: 'HikariCP'
     }
 
-    compileOnly(libs.hadoop2.client) {
+    testImplementation(libs.hive3.standalone.metastore)
+
+    compileOnly(libs.hadoop3.client) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
     }

flink/v1.18/build.gradle

@@ -88,7 +88,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -100,7 +100,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -192,7 +192,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    integrationImplementation(libs.hive2.metastore) {
+    integrationImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -209,7 +209,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.slf4j'
     }
 
-    integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    integrationImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

flink/v1.19/build.gradle

@@ -88,7 +88,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -100,7 +100,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -193,7 +193,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    integrationImplementation(libs.hive2.metastore) {
+    integrationImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -210,7 +210,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.slf4j'
     }
 
-    integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    integrationImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

flink/v1.20/build.gradle

@@ -88,7 +88,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
     // that's really old. We use the core classifier to be able to override our guava
     // version. Luckily, hive-exec seems to work okay so far with this version of guava
     // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
-    testImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    testImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -100,7 +100,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    testImplementation(libs.hive2.metastore) {
+    testImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -193,7 +193,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    integrationImplementation(libs.hive2.metastore) {
+    integrationImplementation(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency
@@ -210,7 +210,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'org.slf4j'
     }
 
-    integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    integrationImplementation("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

gradle.properties

@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=1.20
 systemProp.knownFlinkVersions=1.18,1.19,1.20
-systemProp.defaultHiveVersions=2
-systemProp.knownHiveVersions=2,3
+systemProp.defaultHiveVersions=3
+systemProp.knownHiveVersions=3
 systemProp.defaultSparkVersions=3.5
 systemProp.knownSparkVersions=3.3,3.4,3.5
 systemProp.defaultKafkaVersions=3

gradle/libs.versions.toml

@@ -49,7 +49,6 @@ guava = "33.3.1-jre"
 hadoop2 = "2.7.3"
 hadoop3 = "3.4.1"
 httpcomponents-httpclient5 = "5.4.1"
-hive2 = { strictly = "2.3.9"} # see rich version usage explanation above
 hive3 = "3.1.3"
 immutables-value = "2.10.1"
 jackson-bom = "2.18.1"
@@ -135,12 +134,9 @@ hadoop2-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-c
 hadoop2-minicluster = { module = "org.apache.hadoop:hadoop-minicluster", version.ref = "hadoop2" }
 hadoop3-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop3" }
 hadoop3-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop3" }
-hive2-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive2" }
-hive2-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive2" }
-hive2-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive2" }
-hive2-service = { module = "org.apache.hive:hive-service", version.ref = "hive2" }
 hive3-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive3" }
 hive3-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive3" }
+hive3-standalone-metastore = { module = "org.apache.hive:hive-standalone-metastore", version.ref = "hive3" }
 hive3-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive3" }
 hive3-service = { module = "org.apache.hive:hive-service", version.ref = "hive3" }
 httpcomponents-httpclient5 = { module = "org.apache.httpcomponents.client5:httpclient5", version.ref = "httpcomponents-httpclient5" }

hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java

@@ -509,7 +509,7 @@ public void testRegisterHadoopTableToHiveCatalog() throws IOException, TExceptio
 
     assertThatThrownBy(() -> HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, "table1"))
         .isInstanceOf(NoSuchObjectException.class)
-        .hasMessage("hivedb.table1 table not found");
+        .hasMessage("hive.hivedb.table1 table not found");
     assertThatThrownBy(() -> catalog.loadTable(identifier))
         .isInstanceOf(NoSuchTableException.class)
         .hasMessage("Table does not exist: hivedb.table1");

hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java

@@ -30,12 +30,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Function;
 import org.apache.hadoop.hive.metastore.api.FunctionType;
 import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.thrift.transport.TTransportException;
 import org.junit.jupiter.api.AfterEach;
@@ -121,36 +121,36 @@ public void testGetTablesFailsForNonReconnectableException() throws Exception {
 
   @Test
   public void testExceptionMessages() {
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException(new MetaException("Another meta exception")));
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)
           .hasMessage("Failed to connect to Hive Metastore");
     }
 
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException(new MetaException()));
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)
           .hasMessage("Failed to connect to Hive Metastore");
     }
 
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException());
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)
           .hasMessage("Failed to connect to Hive Metastore");
     }
 
-    try (MockedStatic<MetaStoreUtils> mockedStatic = Mockito.mockStatic(MetaStoreUtils.class)) {
+    try (MockedStatic<JavaUtils> mockedStatic = Mockito.mockStatic(JavaUtils.class)) {
       mockedStatic
-          .when(() -> MetaStoreUtils.newInstance(any(), any(), any()))
+          .when(() -> JavaUtils.newInstance(any(), any(), any()))
           .thenThrow(new RuntimeException("Another instance of Derby may have already booted"));
       assertThatThrownBy(() -> clients.run(client -> client.getTables("default", "t")))
           .isInstanceOf(RuntimeMetaException.class)

kafka-connect/build.gradle

@@ -134,7 +134,7 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {
     implementation 'com.azure:azure-identity'
 
     hive project(':iceberg-hive-metastore')
-    hive(libs.hive2.metastore) {
+    hive(libs.hive3.metastore) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
       exclude group: 'org.pentaho' // missing dependency

mr/build.gradle

@@ -41,7 +41,7 @@ project(':iceberg-mr') {
       exclude group: 'org.apache.avro', module: 'avro'
     }
 
-    compileOnly("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
+    compileOnly("${libs.hive3.exec.get().module}:${libs.hive3.exec.get().getVersion()}:core") {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
       exclude group: 'com.google.guava'
       exclude group: 'com.google.protobuf', module: 'protobuf-java'
@@ -52,8 +52,8 @@ project(':iceberg-mr') {
       exclude group: 'org.pentaho' // missing dependency
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
     }
-    compileOnly libs.hive2.metastore
-    compileOnly libs.hive2.serde
+    compileOnly libs.hive3.metastore
+    compileOnly libs.hive3.serde
 
     implementation libs.caffeine
 
@@ -70,7 +70,7 @@ project(':iceberg-mr') {
     testImplementation libs.kryo.shaded
     testImplementation platform(libs.jackson.bom)
     testImplementation libs.jackson.annotations
-    testImplementation(libs.hive2.service) {
+    testImplementation(libs.hive3.service) {
       exclude group: 'org.apache.hive', module: 'hive-exec'
     }
     testImplementation libs.tez08.dag