From b48c5dd547b9e6295c3be61dcc19ae62963c7eb8 Mon Sep 17 00:00:00 2001 From: Jimmy Lu Date: Mon, 23 Sep 2024 20:07:20 -0700 Subject: [PATCH] Enable reading explicit row number column from Prism connector (#11072) Summary: Pull Request resolved: https://github.com/facebookincubator/velox/pull/11072 Also fix the selective Nimble reader: 1. Add the missing implementation for `FlatMapAsStructColumnReader::estimateMaterializedSize` 2. Fix the performance bug that we refetch the `StripeGroup` for each flatmap node in each stripe Reviewed By: pedroerp Differential Revision: D63151114 fbshipit-source-id: a0b56dbaa4ea5e33324091fe453f35067d4a186b --- velox/dwio/common/ColumnSelector.cpp | 2 +- velox/exec/tests/TableScanTest.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/velox/dwio/common/ColumnSelector.cpp b/velox/dwio/common/ColumnSelector.cpp index ef57fff7ab0f..228da970af4c 100644 --- a/velox/dwio/common/ColumnSelector.cpp +++ b/velox/dwio/common/ColumnSelector.cpp @@ -344,7 +344,7 @@ std::shared_ptr ColumnSelector::fromScanSpec( const RowTypePtr& rowType) { std::vector columnNames; for (auto& child : spec.children()) { - if (child->isConstant()) { + if (child->isConstant() || child->isExplicitRowNumber()) { continue; } std::string name = child->fieldName(); diff --git a/velox/exec/tests/TableScanTest.cpp b/velox/exec/tests/TableScanTest.cpp index fbf5723be09d..19a6a17ae97f 100644 --- a/velox/exec/tests/TableScanTest.cpp +++ b/velox/exec/tests/TableScanTest.cpp @@ -5090,6 +5090,10 @@ TEST_F(TableScanTest, rowNumberInRemainingFilter) { auto file = TempFilePath::create(); writeToFile(file->getPath(), {vector}); auto outputType = ROW({"c1"}, {BIGINT()}); + // FIXME: We should not need r1 in table schema for production code; this is + // just to infer type of r1 in remaining filter expression parser. We + // probably need to find a way to specify the type of r1 without adding it to + // table schema to test this case. For now it's essentially untested. auto schema = ROW({"c1", "r1"}, {BIGINT(), BIGINT()}); auto plan = PlanBuilder() .startTableScan()