upgrade parquet 1.12.0-kylin-r3

Kyligence · Aug 30, 2019 · 851487a · 851487a
1 parent 0e5fb83
commit 851487a
Show file tree

Hide file tree

Showing 42 changed files with 62 additions and 43 deletions.
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/tags/pom.xml b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/core/pom.xml b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/examples/pom.xml b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/external/avro/pom.xml b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/flume/pom.xml b/external/flume/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/graphx/pom.xml b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/launcher/pom.xml b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/mllib/pom.xml b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/pom.xml b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.11</artifactId>
-  <version>2.4.1-kylin-r13</version>
+  <version>2.4.1-kylin-r14</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
@@ -130,7 +130,7 @@
     <!-- Version used for internal directory structure -->
     <hive.version.short>1.2.1</hive.version.short>
     <derby.version>10.12.1.1</derby.version>
-    <parquet.version>1.12.0-kylin-r2</parquet.version>
+    <parquet.version>1.12.0-kylin-r3</parquet.version>
     <fastutil.version>7.0.13</fastutil.version>
     <orc.version>1.5.2</orc.version>
     <orc.classifier>nohive</orc.classifier>

diff --git a/repl/pom.xml b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -351,6 +351,11 @@ object SQLConf {
     val INT96, TIMESTAMP_MICROS, TIMESTAMP_MILLIS = Value
   }
 
+  val PARQUET_CELL_SIZE_LIMIT = buildConf("spark.sql.parquet.cellSizeLimit")
+    .doc(s"Parquet file cell size limit. default 512 * 1024")
+    .longConf
+    .createWithDefault(512 * 1024)
+
   val PARQUET_OUTPUT_TIMESTAMP_TYPE = buildConf("spark.sql.parquet.outputTimestampType")
     .doc("Sets which Parquet timestamp type to use when Spark writes data to Parquet files. " +
       "INT96 is a non-standard but commonly used timestamp type in Parquet. TIMESTAMP_MICROS " +

diff --git a/sql/core/pom.xml b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.4.1-kylin-r13</version>
+    <version>2.4.1-kylin-r14</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/...c/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/...c/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -68,6 +68,8 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
   // Which parquet timestamp type to use when writing.
   private var outputTimestampType: SQLConf.ParquetOutputTimestampType.Value = _
 
+  private val parquetCellSizeLimit = SQLConf.PARQUET_CELL_SIZE_LIMIT.defaultValue.get
+
   // Reusable byte array used to write timestamps as Parquet INT96 values
   private val timestampBuffer = new Array[Byte](12)
 
@@ -115,6 +117,14 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
     }
   }
 
+  @inline private def checkCellSize(cellLength: Long): Unit = {
+    if (cellLength >= parquetCellSizeLimit) {
+      logInfo(s"single cell size: $cellLength ")
+      logInfo(s"spark.sql.parquet.cellSizeLimit: $parquetCellSizeLimit ")
+      this.needCheckRowSize = true
+    }
+  }
+
   private def writeFields(
       row: InternalRow, schema: StructType, fieldWriters: Array[ValueWriter]): Unit = {
     var i = 0
@@ -160,8 +170,10 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
 
       case StringType =>
         (row: SpecializedGetters, ordinal: Int) =>
+          val bytes = row.getUTF8String(ordinal).getBytes
+          checkCellSize(bytes.length)
           recordConsumer.addBinary(
-            Binary.fromReusedByteArray(row.getUTF8String(ordinal).getBytes))
+            Binary.fromReusedByteArray(bytes))
 
       case TimestampType =>
         outputTimestampType match {
@@ -184,7 +196,9 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
 
       case BinaryType =>
         (row: SpecializedGetters, ordinal: Int) =>
-          recordConsumer.addBinary(Binary.fromReusedByteArray(row.getBinary(ordinal)))
+          val bytes = row.getBinary(ordinal)
+          checkCellSize(bytes.length)
+          recordConsumer.addBinary(Binary.fromReusedByteArray(bytes))
 
       case DecimalType.Fixed(precision, scale) =>
         makeDecimalWriter(precision, scale)