diff --git a/velox/dwio/common/DirectDecoder.h b/velox/dwio/common/DirectDecoder.h index 54c46f8032300..c5c0535c5da41 100644 --- a/velox/dwio/common/DirectDecoder.h +++ b/velox/dwio/common/DirectDecoder.h @@ -93,7 +93,7 @@ class DirectDecoder : public IntDecoder { } else if constexpr (std::is_same_v< typename Visitor::DataType, int128_t>) { - if (super::numBytes != 12) { + if (super::numBytes_ != 12) { toSkip = visitor.process(super::template readInt(), atEnd); } else { // Reads INT96 timestamp as int128_t type and extracts the days and diff --git a/velox/dwio/common/IntDecoder.h b/velox/dwio/common/IntDecoder.h index 045da8d4fa100..c61d5af31a9a8 100644 --- a/velox/dwio/common/IntDecoder.h +++ b/velox/dwio/common/IntDecoder.h @@ -441,12 +441,11 @@ inline T IntDecoder::readInt() { return readLittleEndianFromBigEndian(); } else { if constexpr (std::is_same_v) { - if (numBytes == 12) { + if (numBytes_ == 12) { VELOX_DCHECK(!useVInts, "Int96 should not be VInt encoded."); return readInt96(); - } else { - VELOX_NYI(); } + VELOX_NYI(); } return readLongLE(); } diff --git a/velox/dwio/parquet/reader/TimestampColumnReader.h b/velox/dwio/parquet/reader/TimestampColumnReader.h index d7de779f83a16..03bc2dd6f5035 100644 --- a/velox/dwio/parquet/reader/TimestampColumnReader.h +++ b/velox/dwio/parquet/reader/TimestampColumnReader.h @@ -70,7 +70,7 @@ class TimestampColumnReader : public IntegerColumnReader { const RowSet& rows, const uint64_t* /*incomingNulls*/) override { auto& data = formatData_->as(); - // Use int128_t as a workaroud. Timestamp in Velox is of 16-byte length. + // Use int128_t as a workaround. Timestamp in Velox is of 16-byte length. prepareRead(offset, rows, nullptr); readCommon(rows); readOffset_ += rows.back() + 1; diff --git a/velox/dwio/parquet/tests/examples/timestamp_dict_int96.parquet b/velox/dwio/parquet/tests/examples/timestamp_dict_int96.parquet deleted file mode 100644 index 661cb7a285227..0000000000000 Binary files a/velox/dwio/parquet/tests/examples/timestamp_dict_int96.parquet and /dev/null differ diff --git a/velox/dwio/parquet/tests/examples/timestamp_int96.parquet b/velox/dwio/parquet/tests/examples/timestamp_int96_dictionary.parquet similarity index 100% rename from velox/dwio/parquet/tests/examples/timestamp_int96.parquet rename to velox/dwio/parquet/tests/examples/timestamp_int96_dictionary.parquet diff --git a/velox/dwio/parquet/tests/examples/timestamp_int96_plain.parquet b/velox/dwio/parquet/tests/examples/timestamp_int96_plain.parquet new file mode 100644 index 0000000000000..8666703ea732f Binary files /dev/null and b/velox/dwio/parquet/tests/examples/timestamp_int96_plain.parquet differ diff --git a/velox/dwio/parquet/tests/examples/timestamp_plain_int96.parquet b/velox/dwio/parquet/tests/examples/timestamp_plain_int96.parquet deleted file mode 100644 index f2aa666b7d710..0000000000000 Binary files a/velox/dwio/parquet/tests/examples/timestamp_plain_int96.parquet and /dev/null differ diff --git a/velox/dwio/parquet/tests/reader/ParquetTableScanTest.cpp b/velox/dwio/parquet/tests/reader/ParquetTableScanTest.cpp index 0a96227f19867..5a834f06a4538 100644 --- a/velox/dwio/parquet/tests/reader/ParquetTableScanTest.cpp +++ b/velox/dwio/parquet/tests/reader/ParquetTableScanTest.cpp @@ -221,6 +221,72 @@ class ParquetTableScanTest : public HiveConnectorTestBase { writer->close(); } + void testInt96TimestampRead(const std::string& fileName) { + // Timestamp-int96.parquet holds one column (t: TIMESTAMP) and + // 10 rows in one row group. Data is in SNAPPY compressed format. + // The values are: + // |t | + // +-------------------+ + // |2015-06-01 19:34:56| + // |2015-06-02 19:34:56| + // |2001-02-03 03:34:06| + // |1998-03-01 08:01:06| + // |2022-12-23 03:56:01| + // |1980-01-24 00:23:07| + // |1999-12-08 13:39:26| + // |2023-04-21 09:09:34| + // |2000-09-12 22:36:29| + // |2007-12-12 04:27:56| + // +-------------------+ + auto vector = makeFlatVector( + {Timestamp(1433187296, 0), + Timestamp(1433273696, 0), + Timestamp(981171246, 0), + Timestamp(888739266, 0), + Timestamp(1671767761, 0), + Timestamp(317521387, 0), + Timestamp(944660366, 0), + Timestamp(1682068174, 0), + Timestamp(968798189, 0), + Timestamp(1197433676, 0)}); + + loadData( + getExampleFilePath(fileName), + ROW({"t"}, {TIMESTAMP()}), + makeRowVector( + {"t"}, + { + vector, + })); + + assertSelectWithFilter({"t"}, {}, "", "SELECT t from tmp"); + assertSelectWithFilter( + {"t"}, + {}, + "t < TIMESTAMP '2000-09-12 22:36:29'", + "SELECT t from tmp where t < TIMESTAMP '2000-09-12 22:36:29'"); + assertSelectWithFilter( + {"t"}, + {}, + "t <= TIMESTAMP '2000-09-12 22:36:29'", + "SELECT t from tmp where t <= TIMESTAMP '2000-09-12 22:36:29'"); + assertSelectWithFilter( + {"t"}, + {}, + "t > TIMESTAMP '1980-01-24 00:23:07'", + "SELECT t from tmp where t > TIMESTAMP '1980-01-24 00:23:07'"); + assertSelectWithFilter( + {"t"}, + {}, + "t >= TIMESTAMP '1980-01-24 00:23:07'", + "SELECT t from tmp where t >= TIMESTAMP '1980-01-24 00:23:07'"); + assertSelectWithFilter( + {"t"}, + {}, + "t == TIMESTAMP '2022-12-23 03:56:01'", + "SELECT t from tmp where t == TIMESTAMP '2022-12-23 03:56:01'"); + } + private: RowTypePtr getRowType(std::vector&& outputColumnNames) const { std::vector types; @@ -719,70 +785,12 @@ TEST_F(ParquetTableScanTest, sessionTimezone) { assertSelectWithTimezone({"a"}, "SELECT a FROM tmp", "Asia/Shanghai"); } -TEST_F(ParquetTableScanTest, timestampFilter) { - // Timestamp-int96.parquet holds one column (t: TIMESTAMP) and - // 10 rows in one row group. Data is in SNAPPY compressed format. - // The values are: - // |t | - // +-------------------+ - // |2015-06-01 19:34:56| - // |2015-06-02 19:34:56| - // |2001-02-03 03:34:06| - // |1998-03-01 08:01:06| - // |2022-12-23 03:56:01| - // |1980-01-24 00:23:07| - // |1999-12-08 13:39:26| - // |2023-04-21 09:09:34| - // |2000-09-12 22:36:29| - // |2007-12-12 04:27:56| - // +-------------------+ - auto vector = makeFlatVector( - {Timestamp(1433187296, 0), - Timestamp(1433273696, 0), - Timestamp(981171246, 0), - Timestamp(888739266, 0), - Timestamp(1671767761, 0), - Timestamp(317521387, 0), - Timestamp(944660366, 0), - Timestamp(1682068174, 0), - Timestamp(968798189, 0), - Timestamp(1197433676, 0)}); - - loadData( - getExampleFilePath("timestamp_int96.parquet"), - ROW({"t"}, {TIMESTAMP()}), - makeRowVector( - {"t"}, - { - vector, - })); +TEST_F(ParquetTableScanTest, timestampInt96Dictionary) { + testInt96TimestampRead("timestamp_int96_dictionary.parquet"); +} - assertSelectWithFilter({"t"}, {}, "", "SELECT t from tmp"); - assertSelectWithFilter( - {"t"}, - {}, - "t < TIMESTAMP '2000-09-12 22:36:29'", - "SELECT t from tmp where t < TIMESTAMP '2000-09-12 22:36:29'"); - assertSelectWithFilter( - {"t"}, - {}, - "t <= TIMESTAMP '2000-09-12 22:36:29'", - "SELECT t from tmp where t <= TIMESTAMP '2000-09-12 22:36:29'"); - assertSelectWithFilter( - {"t"}, - {}, - "t > TIMESTAMP '1980-01-24 00:23:07'", - "SELECT t from tmp where t > TIMESTAMP '1980-01-24 00:23:07'"); - assertSelectWithFilter( - {"t"}, - {}, - "t >= TIMESTAMP '1980-01-24 00:23:07'", - "SELECT t from tmp where t >= TIMESTAMP '1980-01-24 00:23:07'"); - assertSelectWithFilter( - {"t"}, - {}, - "t == TIMESTAMP '2022-12-23 03:56:01'", - "SELECT t from tmp where t == TIMESTAMP '2022-12-23 03:56:01'"); +TEST_F(ParquetTableScanTest, timestampInt96Plain) { + testInt96TimestampRead("timestamp_int96_plain.parquet"); } TEST_F(ParquetTableScanTest, timestampPrecisionMicrosecond) { @@ -836,34 +844,6 @@ TEST_F(ParquetTableScanTest, timestampPrecisionMicrosecond) { assertEqualResults({expected}, result.second); } - -TEST_F(ParquetTableScanTest, timestampINT96) { - auto a = makeFlatVector({Timestamp(1, 0), Timestamp(2, 0)}); - auto expected = makeRowVector({"time"}, {a}); - createDuckDbTable("expected", {expected}); - - auto vector = makeArrayVector({{}}); - loadData( - getExampleFilePath("timestamp_dict_int96.parquet"), - ROW({"time"}, {TIMESTAMP()}), - makeRowVector( - {"time"}, - { - vector, - })); - assertSelect({"time"}, "SELECT time from expected"); - - loadData( - getExampleFilePath("timestamp_plain_int96.parquet"), - ROW({"time"}, {TIMESTAMP()}), - makeRowVector( - {"time"}, - { - vector, - })); - assertSelect({"time"}, "SELECT time from expected"); -} - int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); folly::Init init{&argc, &argv, false};