corss verify hive engine on hive 2.3

apache · Nov 24, 2023 · 8ca02a3 · 8ca02a3
1 parent 4d48796
commit 8ca02a3
Show file tree

Hide file tree

Showing 5 changed files with 76 additions and 40 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -242,7 +242,12 @@ jobs:
       matrix:
         java:
           - 8
+        hive-archive: [ "" ]
         comment: [ "normal" ]
+        include:
+          - java: 8
+            hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.9 -Dhive.archive.name=apache-hive-2.3.9-bin.tar.gz'
+            comment: 'verify-on-hive-2.3-binary'
     steps:
       - uses: actions/checkout@v3
       - name: Tune Runner VM
@@ -261,8 +266,15 @@ jobs:
       - name: Build and test Hive with maven w/o linters
         run: |
           TEST_MODULES="externals/kyuubi-hive-sql-engine,integration-tests/kyuubi-hive-it"
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test
+          ./build/mvn ${MVN_OPT} ${{ matrix.hive-archive }} -pl ${TEST_MODULES} -am clean install -DskipTests
+          # Hive 2.3.9 ships Derby 10.10.2.0, which may fail to boostrap on latest JDK 8
+          # https://github.com/apache/hive/pull/4895
+          if [[ "${{ matrix.hive-archive }}" == *apache-hive-2.3.9-bin.tar.gz* ]]; then
+            HIVE_239_LIB="$PWD/externals/kyuubi-download/target/apache-hive-2.3.9-bin/lib"
+            rm $HIVE_239_LIB/derby-*
+            wget https://repo1.maven.org/maven2/org/apache/derby/derby/10.14.2.0/derby-10.14.2.0.jar -P $HIVE_239_LIB
+          fi
+          ./build/mvn ${MVN_OPT} ${{ matrix.hive-archive }} -pl ${TEST_MODULES} test
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v3

diff --git a/externals/kyuubi-hive-sql-engine/pom.xml b/externals/kyuubi-hive-sql-engine/pom.xml
@@ -50,18 +50,6 @@
             <version>${project.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.hive</groupId>
-            <artifactId>hive-service-rpc</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.thrift</groupId>
-            <artifactId>libfb303</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
         <dependency>
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>
@@ -73,12 +61,6 @@
             <artifactId>commons-collections</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.thrift</groupId>
-            <artifactId>libthrift</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>failureaccess</artifactId>
@@ -206,6 +188,18 @@
                             </excludes>
                         </filter>
                     </filters>
+                    <relocations>
+                        <relocation>
+                            <pattern>com.fasterxml.jackson</pattern>
+                            <shadedPattern>${kyuubi.shade.packageName}.com.fasterxml.jackson</shadedPattern>
+                            <includes>
+                                <include>com.fasterxml.jackson.**</include>
+                            </includes>
+                        </relocation>
+                    </relocations>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
+                    </transformers>
                 </configuration>
                 <executions>
                     <execution>

diff --git a/...-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionManager.scala b/...-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionManager.scala
@@ -18,6 +18,7 @@
 package org.apache.kyuubi.engine.hive.session
 
 import java.io.File
+import java.util.{List => JList}
 import java.util.concurrent.Future
 
 import scala.collection.JavaConverters._
@@ -34,6 +35,7 @@ import org.apache.kyuubi.engine.hive.HiveSQLEngine
 import org.apache.kyuubi.engine.hive.operation.HiveOperationManager
 import org.apache.kyuubi.operation.OperationManager
 import org.apache.kyuubi.session.{Session, SessionHandle, SessionManager}
+import org.apache.kyuubi.util.reflect.DynConstructors
 
 class HiveSessionManager(engine: HiveSQLEngine) extends SessionManager("HiveSessionManager") {
   override protected def isServer: Boolean = false
@@ -78,15 +80,37 @@ class HiveSessionManager(engine: HiveSQLEngine) extends SessionManager("HiveSess
       val sessionHandle =
         conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).getOrElse(SessionHandle())
       val hive = {
-        val sessionWithUGI = new ImportedHiveSessionImpl(
-          new ImportedSessionHandle(sessionHandle.toTSessionHandle, protocol),
-          protocol,
-          user,
-          password,
-          HiveSQLEngine.hiveConf,
-          ipAddress,
-          null,
-          Seq(ipAddress).asJava)
+
+        val sessionWithUGI = DynConstructors.builder()
+          .impl( // for Hive 3.1
+            classOf[ImportedHiveSessionImpl],
+            classOf[ImportedSessionHandle],
+            classOf[TProtocolVersion],
+            classOf[String],
+            classOf[String],
+            classOf[HiveConf],
+            classOf[String],
+            classOf[String],
+            classOf[JList[String]])
+          .impl( // for Hive 2.3
+            classOf[ImportedHiveSessionImpl],
+            classOf[ImportedSessionHandle],
+            classOf[TProtocolVersion],
+            classOf[String],
+            classOf[String],
+            classOf[HiveConf],
+            classOf[String],
+            classOf[String])
+          .build[ImportedHiveSessionImpl]()
+          .newInstance(
+            new ImportedSessionHandle(sessionHandle.toTSessionHandle, protocol),
+            protocol,
+            user,
+            password,
+            HiveSQLEngine.hiveConf,
+            ipAddress,
+            null,
+            Seq(ipAddress).asJava)
         val proxy = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi)
         sessionWithUGI.setProxySession(proxy)
         proxy

diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/JdbcUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/JdbcUtils.scala
@@ -98,6 +98,12 @@ object JdbcUtils extends Logging {
     }
   }
 
+  def mapResultSet[R](rs: ResultSet)(rowMapper: ResultSet => R): Seq[R] = {
+    val builder = Seq.newBuilder[R]
+    while (rs.next()) builder += rowMapper(rs)
+    builder.result
+  }
+
   def redactPassword(password: Option[String]): String = {
     password match {
       case Some(s) if StringUtils.isNotBlank(s) => s"${"*" * s.length}(length:${s.length})"

diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/HiveEngineTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/HiveEngineTests.scala
@@ -23,6 +23,7 @@ import org.apache.commons.lang3.{JavaVersion, SystemUtils}
 
 import org.apache.kyuubi.operation.HiveJDBCTestHelper
 import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
+import org.apache.kyuubi.util.JdbcUtils
 
 /**
  * hive tests disabled for JAVA 11
@@ -229,14 +230,11 @@ trait HiveEngineTests extends HiveJDBCTestHelper {
     assume(SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_1_8))
     withJdbcStatement() { statement =>
       val resultSet = statement.getConnection.getMetaData.getTableTypes
-      val expected = Set("TABLE", "VIEW", "MATERIALIZED_VIEW")
-      var tableTypes = Set[String]()
-      while (resultSet.next()) {
-        assert(expected.contains(resultSet.getString(TABLE_TYPE)))
-        tableTypes += resultSet.getString(TABLE_TYPE)
-      }
-      assert(!resultSet.next())
-      assert(expected.size === tableTypes.size)
+      // Hive3 removes support for INDEX_TABLE
+      val hive2Expected = Set("TABLE", "VIEW", "MATERIALIZED_VIEW", "INDEX_TABLE")
+      val hive3Expected = Set("TABLE", "VIEW", "MATERIALIZED_VIEW")
+      val tableTypes = JdbcUtils.mapResultSet(resultSet) { rs => rs.getString(TABLE_TYPE) }.toSet
+      assert(tableTypes === hive2Expected || tableTypes === hive3Expected)
     }
   }
 
@@ -387,10 +385,12 @@ trait HiveEngineTests extends HiveJDBCTestHelper {
       assert(typeInfo.getInt(DATA_TYPE) === java.sql.Types.TIMESTAMP)
 
       typeInfo.next()
-      assert(typeInfo.getString(TYPE_NAME) === "TIMESTAMP WITH LOCAL TIME ZONE")
-      assert(typeInfo.getInt(DATA_TYPE) === java.sql.Types.OTHER)
+      // Hive3 supports TIMESTAMP WITH LOCAL TIME ZONE
+      if (typeInfo.getString(TYPE_NAME) == "TIMESTAMP WITH LOCAL TIME ZONE") {
+        assert(typeInfo.getInt(DATA_TYPE) === java.sql.Types.OTHER)
+        typeInfo.next()
+      }
 
-      typeInfo.next()
       assert(typeInfo.getString(TYPE_NAME) === "INTERVAL_YEAR_MONTH")
       assert(typeInfo.getInt(DATA_TYPE) === java.sql.Types.OTHER)