Add PartitionHelper

yhcast0 · Dec 8, 2023 · 1d99d17 · 1d99d17
1 parent 80e070e
commit 1d99d17
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 18 deletions.
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
@@ -96,6 +96,7 @@
       <version>${protobuf.version}</version>
     </dependency>
 -->
+
     <dependency>
       <groupId>${hive.group}</groupId>
       <artifactId>hive-common</artifactId>
@@ -126,18 +127,6 @@
       <artifactId>hive-shims</artifactId>
       <scope>${hive.shims.scope}</scope>
     </dependency>
-    <dependency>
-      <groupId>io.transwarp.inceptor</groupId>
-      <artifactId>inceptor-shaded</artifactId>
-      <version>1.0-SNAPSHOT</version>
-      <scope>${hive.serde.scope}</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.hive</groupId>
-          <artifactId>inceptor-serde</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-llap-common</artifactId>
@@ -153,6 +142,26 @@
       <artifactId>hive-contrib</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>io.transwarp.inceptor</groupId>
+      <artifactId>inceptor-shaded</artifactId>
+      <version>1.0-SNAPSHOT</version>
+      <scope>${hive.serde.scope}</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>inceptor-serde</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>inceptor-exec</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>inceptor-metastore</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <!-- hive-serde already depends on avro, but this brings in customized config of avro deps from parent -->
     <dependency>
       <groupId>org.apache.avro</groupId>

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.hive
 
+import io.kyligence.compact.inceptor.PartitionHelper
 import io.transwarp.hive.shaded.serde2.objectinspector.ObjectInspectorConverters
 import io.transwarp.hive.shaded.serde2.objectinspector.primitive.DateObjectInspector
 import org.apache.hadoop.conf.Configuration
@@ -215,15 +216,15 @@ class HadoopTableReader(
 
     val hivePartitionRDDs = verifyPartitionPath(partitionToDeserializer)
       .map { case (partition, partDeserializer) =>
-      val partDesc = Utilities.getPartitionDescFromTableDesc(tableDesc, partition, true)
+      val partDesc = Utilities.getPartitionDesc(partition)
       var partPath = partition.getDataLocation
       val inputPathStr = applyFilterIfNeeded(partPath, filterOpt)
       val skipHeaderLineCount =
         tableDesc.getProperties.getProperty("skip.header.line.count", "0").toInt
       val isTextInputFormatTable =
         classOf[TextInputFormat].isAssignableFrom(partDesc.getInputFileFormatClass)
       // Get partition field info
-      val partSpec = partDesc.getPartSpec
+      val partSpec = PartitionHelper.getPartSpec(partDesc)
       val partProps = partDesc.getProperties
 
       val partColsDelimited: String = partProps.getProperty(META_TABLE_PARTITION_COLUMNS)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -17,17 +17,17 @@
 
 package org.apache.spark.sql.hive.client
 
+import io.kyligence.compact.inceptor.PartitionHelper
+
 import java.io.PrintStream
 import java.lang.{Iterable => JIterable}
 import java.lang.reflect.InvocationTargetException
 import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Locale, Map => JMap}
 import java.util.concurrent.TimeUnit._
-
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
-
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hive.common.StatsSetupConst
@@ -44,7 +44,6 @@ import org.apache.hadoop.hive.serde.serdeConstants
 import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 import org.apache.hadoop.security.UserGroupInformation
-
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.source.HiveCatalogMetrics
@@ -1174,7 +1173,8 @@ private[hive] object HiveClientImpl extends Logging {
       Map.empty
     }
     CatalogTablePartition(
-      spec = Option(hp.getSpec).map(_.asScala.toMap).getOrElse(Map.empty),
+      spec = Option(PartitionHelper.getSpec(hp.getTable, hp.getTPartition))
+        .map(_.asScala.toMap).getOrElse(Map.empty),
       storage = CatalogStorageFormat(
         locationUri = Option(CatalogUtils.stringToURI(apiPartition.getSd.getLocation)),
         inputFormat = Option(apiPartition.getSd.getInputFormat),