KE-41399 Avoid parquet footer reads twice in vectorized reader (#71)

KE-41399 Avoid parquet footer reads twice in vectorized reader
Kyligence · May 22, 2023 · f4b689b · f4b689b
1 parent 1f2ac54
commit f4b689b
Show file tree

Hide file tree

Showing 23 changed files with 40 additions and 36 deletions.
diff --git a/parquet-arrow/pom.xml b/parquet-arrow/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-cascading-deprecated/pom.xml b/parquet-cascading-deprecated/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-cascading3-deprecated/pom.xml b/parquet-cascading3-deprecated/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml
@@ -24,7 +24,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <artifactId>parquet-format-structures</artifactId>

diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
@@ -663,12 +663,12 @@ public static ParquetFileReader open(InputFile file, ParquetReadOptions options)
 
   protected final SeekableInputStream f;
   private final InputFile file;
-  private final ParquetReadOptions options;
+  private ParquetReadOptions options;
   private final Map<ColumnPath, ColumnDescriptor> paths = new HashMap<>();
   private final FileMetaData fileMetaData; // may be null
-  private final List<BlockMetaData> blocks;
-  private final List<ColumnIndexStore> blockIndexStores;
-  private final List<RowRanges> blockRowRanges;
+  private List<BlockMetaData> blocks;
+  private List<ColumnIndexStore> blockIndexStores;
+  private List<RowRanges> blockRowRanges;
 
   // not final. in some cases, this may be lazily loaded for backward-compat.
   private ParquetMetadata footer;
@@ -754,17 +754,8 @@ public ParquetFileReader(Configuration conf, Path file, ParquetMetadata footer)
     this.f = this.file.newStream();
     this.fileMetaData = footer.getFileMetaData();
     this.fileDecryptor = fileMetaData.getFileDecryptor();
-    if (null == fileDecryptor) {
-      this.options = HadoopReadOptions.builder(conf).build();
-    } else {
-      this.options = HadoopReadOptions.builder(conf)
-                                      .withDecryption(fileDecryptor.getDecryptionProperties())
-                                      .build();
-    }
     this.footer = footer;
-    this.blocks = filterRowGroups(footer.getBlocks());
-    this.blockIndexStores = listWithNulls(this.blocks.size());
-    this.blockRowRanges = listWithNulls(this.blocks.size());
+    resetBlocks(conf);
     for (ColumnDescriptor col : footer.getFileMetaData().getSchema().getColumns()) {
       paths.put(ColumnPath.get(col.getPath()), col);
     }
@@ -808,6 +799,19 @@ public ParquetFileReader(InputFile file, ParquetReadOptions options) throws IOEx
     }
   }
 
+  public void resetBlocks(Configuration conf) throws IOException {
+    if (null == fileDecryptor) {
+      this.options = HadoopReadOptions.builder(conf).build();
+    } else {
+      this.options = HadoopReadOptions.builder(conf)
+        .withDecryption(fileDecryptor.getDecryptionProperties())
+        .build();
+    }
+    this.blocks = filterRowGroups(footer.getBlocks());
+    this.blockIndexStores = listWithNulls(this.blocks.size());
+    this.blockRowRanges = listWithNulls(this.blocks.size());
+  }
+
   private static <T> List<T> listWithNulls(int size) {
     return Stream.generate(() -> (T) null).limit(size).collect(Collectors.toCollection(ArrayList<T>::new));
   }

diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-scrooge-deprecated/pom.xml b/parquet-scrooge-deprecated/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/parquet-tools-deprecated/pom.xml b/parquet-tools-deprecated/pom.xml
@@ -21,7 +21,7 @@
     <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>1.12.2-kylin-r5</version>
+    <version>1.12.2-kylin-r6</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>

diff --git a/pom.xml b/pom.xml
@@ -9,7 +9,7 @@
 
   <groupId>org.apache.parquet</groupId>
   <artifactId>parquet</artifactId>
-  <version>1.12.2-kylin-r5</version>
+  <version>1.12.2-kylin-r6</version>
   <packaging>pom</packaging>
 
   <name>Apache Parquet MR</name>