Support struct column reading with different schemas (5962)

velox/connectors/hive/SplitReader.cpp

@@ -220,9 +220,18 @@ std::vector<TypePtr> SplitReader::adaptColumns(
     } else {
       auto fileTypeIdx = fileType->getChildIdxIfExists(fieldName);
       if (!fileTypeIdx.has_value()) {
-        // Column is missing. Most likely due to schema evolution.
-        VELOX_CHECK(tableSchema);
-        setNullConstantValue(childSpec, tableSchema->findChild(fieldName));
+        // If field name exists in the user-specified output type,
+        // set the column as null constant.
+        // Related PR: https://github.com/facebookincubator/velox/pull/6427.
+        auto outputTypeIdx = readerOutputType_->getChildIdxIfExists(fieldName);
+        if (outputTypeIdx.has_value()) {
+          setNullConstantValue(
+              childSpec, readerOutputType_->childAt(outputTypeIdx.value()));
+        } else {
+          // Column is missing. Most likely due to schema evolution.
+          VELOX_CHECK(tableSchema);
+          setNullConstantValue(childSpec, tableSchema->findChild(fieldName));
+        }
       } else {
         // Column no longer missing, reset constant value set on the spec.
         childSpec->setConstantValue(nullptr);

velox/dwio/common/SelectiveStructColumnReader.cpp

@@ -132,8 +132,7 @@ void SelectiveStructColumnReaderBase::read(
     activeRows = outputRows_;
   }
 
-  auto& childSpecs = scanSpec_->children();
-  VELOX_CHECK(!childSpecs.empty());
+  auto& childSpecs = scanSpec_->stableChildren();
   for (size_t i = 0; i < childSpecs.size(); ++i) {
     auto& childSpec = childSpecs[i];
     if (isChildConstant(*childSpec)) {
@@ -218,7 +217,7 @@ bool SelectiveStructColumnReaderBase::isChildConstant(
        fileType_->type()->kind() !=
            TypeKind::MAP && // If this is the case it means this is a flat map,
                             // so it can't have "missing" fields.
-       childSpec.channel() >= fileType_->size());
+       !fileType_->containsChild(childSpec.fieldName()));
 }
 
 namespace {
@@ -298,7 +297,6 @@ void setNullField(vector_size_t size, VectorPtr& field) {
 void SelectiveStructColumnReaderBase::getValues(
     RowSet rows,
     VectorPtr* result) {
-  VELOX_CHECK(!scanSpec_->children().empty());
   VELOX_CHECK(
       *result != nullptr,
       "SelectiveStructColumnReaderBase expects a non-null result");
@@ -335,7 +333,7 @@ void SelectiveStructColumnReaderBase::getValues(
     resultRow->clearNulls(0, rows.size());
   }
   bool lazyPrepared = false;
-  for (auto& childSpec : scanSpec_->children()) {
+  for (auto& childSpec : scanSpec_->stableChildren()) {
     if (!childSpec->projectOut()) {
       continue;
     }

velox/dwio/common/TypeWithId.h

@@ -59,6 +59,11 @@ class TypeWithId : public velox::Tree<std::shared_ptr<const TypeWithId>> {
 
   const std::shared_ptr<const TypeWithId>& childAt(uint32_t idx) const override;
 
+  bool containsChild(const std::string& name) const {
+    VELOX_CHECK_EQ(type_->kind(), velox::TypeKind::ROW);
+    return type_->as<velox::TypeKind::ROW>().containsChild(name);
+  }
+
   const std::shared_ptr<const TypeWithId>& childByName(
       const std::string& name) const {
     VELOX_CHECK_EQ(type_->kind(), velox::TypeKind::ROW);

velox/dwio/parquet/reader/ParquetColumnReader.cpp

@@ -38,7 +38,8 @@ std::unique_ptr<dwio::common::SelectiveColumnReader> ParquetColumnReader::build(
     const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
     const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
     ParquetParams& params,
-    common::ScanSpec& scanSpec) {
+    common::ScanSpec& scanSpec,
+    memory::MemoryPool& pool) {
   auto colName = scanSpec.fieldName();
 
   switch (dataType->type()->kind()) {
@@ -59,19 +60,19 @@ std::unique_ptr<dwio::common::SelectiveColumnReader> ParquetColumnReader::build(
 
     case TypeKind::ROW:
       return std::make_unique<StructColumnReader>(
-          requestedType, dataType, params, scanSpec);
+          requestedType, dataType, params, scanSpec, pool);
 
     case TypeKind::VARBINARY:
     case TypeKind::VARCHAR:
       return std::make_unique<StringColumnReader>(dataType, params, scanSpec);
 
     case TypeKind::ARRAY:
       return std::make_unique<ListColumnReader>(
-          requestedType, dataType, params, scanSpec);
+          requestedType, dataType, params, scanSpec, pool);
 
     case TypeKind::MAP:
       return std::make_unique<MapColumnReader>(
-          requestedType, dataType, params, scanSpec);
+          requestedType, dataType, params, scanSpec, pool);
 
     case TypeKind::BOOLEAN:
       return std::make_unique<BooleanColumnReader>(

velox/dwio/parquet/reader/ParquetColumnReader.h

@@ -45,6 +45,7 @@ class ParquetColumnReader {
       const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
       const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
       ParquetParams& params,
-      common::ScanSpec& scanSpec);
+      common::ScanSpec& scanSpec,
+      memory::MemoryPool& pool);
 };
 } // namespace facebook::velox::parquet

velox/dwio/parquet/reader/ParquetReader.cpp

@@ -83,6 +83,11 @@ class ReaderBase {
   /// the data still exists in the buffered inputs.
   bool isRowGroupBuffered(int32_t rowGroupIndex) const;
 
+  static std::shared_ptr<const dwio::common::TypeWithId> createTypeWithId(
+      const std::shared_ptr<const dwio::common::TypeWithId>& inputType,
+      const RowTypePtr& rowTypePtr,
+      bool fileColumnNamesReadAsLowerCase);
+
  private:
   // Reads and parses file footer.
   void loadFileMetaData();
@@ -563,6 +568,33 @@ std::shared_ptr<const RowType> ReaderBase::createRowType(
       std::move(childNames), std::move(childTypes));
 }
 
+std::shared_ptr<const dwio::common::TypeWithId> ReaderBase::createTypeWithId(
+    const std::shared_ptr<const dwio::common::TypeWithId>& inputType,
+    const RowTypePtr& rowTypePtr,
+    bool fileColumnNamesReadAsLowerCase) {
+  if (!fileColumnNamesReadAsLowerCase) {
+    return inputType;
+  }
+  std::vector<std::string> names;
+  names.reserve(rowTypePtr->names().size());
+  std::vector<TypePtr> types = rowTypePtr->children();
+  for (const auto& name : rowTypePtr->names()) {
+    std::string childName = name;
+    folly::toLowerAscii(childName);
+    names.emplace_back(childName);
+  }
+  auto convertedType =
+      TypeFactory<TypeKind::ROW>::create(std::move(names), std::move(types));
+
+  auto children = inputType->getChildren();
+  return std::make_shared<const dwio::common::TypeWithId>(
+      convertedType,
+      std::move(children),
+      inputType->id(),
+      inputType->maxId(),
+      inputType->column());
+}
+
 void ReaderBase::scheduleRowGroups(
     const std::vector<uint32_t>& rowGroupIds,
     int32_t currentGroup,
@@ -639,10 +671,14 @@ ParquetRowReader::ParquetRowReader(
   ParquetParams params(pool_, columnReaderStats_, readerBase_->fileMetaData());
   auto columnSelector = options_.getSelector();
   columnReader_ = ParquetColumnReader::build(
-      columnSelector->getSchemaWithId(),
+      ReaderBase::createTypeWithId(
+          columnSelector->getSchemaWithId(),
+          asRowType(options_.getSelector()->getSchemaWithId()->type()),
+          readerBase_->isFileColumnNamesReadAsLowerCase()),
       readerBase_->schemaWithId(), // Id is schema id
       params,
-      *options_.getScanSpec());
+      *options_.getScanSpec(),
+      pool_);
 
   filterRowGroups();
   if (!rowGroupIds_.empty()) {

velox/dwio/parquet/reader/RepeatedColumnReader.cpp

@@ -31,6 +31,9 @@ PageReader* FOLLY_NULLABLE readLeafRepDefs(
       return nullptr;
     }
     auto pageReader = reader->formatData().as<ParquetData>().reader();
+    if (pageReader == nullptr) {
+      return nullptr;
+    }
     pageReader->decodeRepDefs(numTop);
     return pageReader;
   }
@@ -111,7 +114,8 @@ MapColumnReader::MapColumnReader(
     const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
     const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
     ParquetParams& params,
-    common::ScanSpec& scanSpec)
+    common::ScanSpec& scanSpec,
+    memory::MemoryPool& pool)
     : dwio::common::SelectiveMapColumnReader(
           requestedType,
           dataType,
@@ -121,9 +125,17 @@ MapColumnReader::MapColumnReader(
   auto& keyChildType = requestedType->childAt(0);
   auto& elementChildType = requestedType->childAt(1);
   keyReader_ = ParquetColumnReader::build(
-      keyChildType, fileType_->childAt(0), params, *scanSpec.children()[0]);
+      keyChildType,
+      fileType_->childAt(0),
+      params,
+      *scanSpec.children()[0],
+      pool);
   elementReader_ = ParquetColumnReader::build(
-      elementChildType, fileType_->childAt(1), params, *scanSpec.children()[1]);
+      elementChildType,
+      fileType_->childAt(1),
+      params,
+      *scanSpec.children()[1],
+      pool);
   reinterpret_cast<const ParquetTypeWithId*>(dataType.get())
       ->makeLevelInfo(levelInfo_);
   children_ = {keyReader_.get(), elementReader_.get()};
@@ -221,15 +233,16 @@ ListColumnReader::ListColumnReader(
     const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
     const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
     ParquetParams& params,
-    common::ScanSpec& scanSpec)
+    common::ScanSpec& scanSpec,
+    memory::MemoryPool& pool)
     : dwio::common::SelectiveListColumnReader(
           requestedType,
           dataType,
           params,
           scanSpec) {
   auto& childType = requestedType->childAt(0);
   child_ = ParquetColumnReader::build(
-      childType, fileType_->childAt(0), params, *scanSpec.children()[0]);
+      childType, fileType_->childAt(0), params, *scanSpec.children()[0], pool);
   reinterpret_cast<const ParquetTypeWithId*>(dataType.get())
       ->makeLevelInfo(levelInfo_);
   children_ = {child_.get()};

velox/dwio/parquet/reader/RepeatedColumnReader.h

@@ -59,7 +59,8 @@ class MapColumnReader : public dwio::common::SelectiveMapColumnReader {
       const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
       const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
       ParquetParams& params,
-      common::ScanSpec& scanSpec);
+      common::ScanSpec& scanSpec,
+      memory::MemoryPool& pool);
 
   void prepareRead(
       vector_size_t offset,
@@ -115,7 +116,8 @@ class ListColumnReader : public dwio::common::SelectiveListColumnReader {
       const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
       const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
       ParquetParams& params,
-      common::ScanSpec& scanSpec);
+      common::ScanSpec& scanSpec,
+      memory::MemoryPool& pool);
 
   void prepareRead(
       vector_size_t offset,

velox/dwio/parquet/reader/StructColumnReader.cpp

@@ -23,21 +23,46 @@ StructColumnReader::StructColumnReader(
     const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
     const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
     ParquetParams& params,
-    common::ScanSpec& scanSpec)
+    common::ScanSpec& scanSpec,
+    memory::MemoryPool& pool)
     : SelectiveStructColumnReader(requestedType, dataType, params, scanSpec) {
   auto& childSpecs = scanSpec_->stableChildren();
+  std::vector<int> missingFields;
   for (auto i = 0; i < childSpecs.size(); ++i) {
     auto childSpec = childSpecs[i];
     if (childSpecs[i]->isConstant()) {
       continue;
     }
-    auto childDataType = fileType_->childByName(childSpec->fieldName());
-    auto childRequestedType =
-        requestedType_->childByName(childSpec->fieldName());
+    const auto& fieldName = childSpecs[i]->fieldName();
+    if (!fileType_->containsChild(fieldName)) {
+      missingFields.emplace_back(i);
+      continue;
+    }
+    auto childDataType = fileType_->childByName(fieldName);
+    auto childRequestedType = requestedType_->childByName(fieldName);
     addChild(ParquetColumnReader::build(
-        childRequestedType, childDataType, params, *childSpec));
+        childRequestedType, childDataType, params, *childSpec, pool));
     childSpecs[i]->setSubscript(children_.size() - 1);
   }
+
+  if (missingFields.size() > 0) {
+    // Set the struct as null if all the children fields in the output type are
+    // missing and the number of child fields is more than one.
+    if (childSpecs.size() > 1 && missingFields.size() == childSpecs.size()) {
+      scanSpec_->setConstantValue(
+          BaseVector::createNullConstant(requestedType_->type(), 1, &pool));
+    } else {
+      // Set null constant for the missing child field of output type.
+      for (int channel : missingFields) {
+        childSpecs[channel]->setConstantValue(BaseVector::createNullConstant(
+            requestedType_->childByName(childSpecs[channel]->fieldName())
+                ->type(),
+            1,
+            &pool));
+      }
+    }
+  }
+
   auto type = reinterpret_cast<const ParquetTypeWithId*>(fileType_.get());
   if (type->parent()) {
     levelMode_ = reinterpret_cast<const ParquetTypeWithId*>(fileType_.get())
@@ -47,7 +72,10 @@ StructColumnReader::StructColumnReader(
     // this and the child.
     auto child = childForRepDefs_;
     for (;;) {
-      assert(child);
+      if (child == nullptr) {
+        levelMode_ = LevelMode::kNulls;
+        break;
+      }
       if (child->fileType().type()->kind() == TypeKind::ARRAY ||
           child->fileType().type()->kind() == TypeKind::MAP) {
         levelMode_ = LevelMode::kStructOverLists;

velox/dwio/parquet/reader/StructColumnReader.h

@@ -27,7 +27,8 @@ class StructColumnReader : public dwio::common::SelectiveStructColumnReader {
       const std::shared_ptr<const dwio::common::TypeWithId>& requestedType,
       const std::shared_ptr<const dwio::common::TypeWithId>& dataType,
       ParquetParams& params,
-      common::ScanSpec& scanSpec);
+      common::ScanSpec& scanSpec,
+      memory::MemoryPool& pool);
 
   void read(vector_size_t offset, RowSet rows, const uint64_t* incomingNulls)
       override;