Skip to content

Commit

Permalink
Remove duplicate code
Browse files Browse the repository at this point in the history
  • Loading branch information
@rui-mo
rui-mo committed Nov 11, 2024
1 parent 65d4e7e commit 316bf01
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 115 deletions.
69 changes: 43 additions & 26 deletions velox/dwio/common/SelectiveIntegerColumnReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,23 +127,26 @@ void SelectiveIntegerColumnReader::processFilter(
ExtractValues extractValues,
const RowSet& rows) {
if (filter == nullptr) {
readHelper<Reader, velox::common::AlwaysTrue, isDense>(
&dwio::common::alwaysTrue(), rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<Reader, velox::common::AlwaysTrue, isDense>(
&dwio::common::alwaysTrue(), rows, extractValues);
return;
}

switch (filter->kind()) {
case velox::common::FilterKind::kAlwaysTrue:
readHelper<Reader, velox::common::AlwaysTrue, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<Reader, velox::common::AlwaysTrue, isDense>(
filter, rows, extractValues);
break;
case velox::common::FilterKind::kIsNull:
if constexpr (kEncodingHasNulls) {
filterNulls<int64_t>(
rows, true, !std::is_same_v<decltype(extractValues), DropValues>);
} else {
readHelper<Reader, velox::common::IsNull, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<Reader, velox::common::IsNull, isDense>(
filter, rows, extractValues);
}
break;
case velox::common::FilterKind::kIsNotNull:
Expand All @@ -152,41 +155,55 @@ void SelectiveIntegerColumnReader::processFilter(
std::is_same_v<decltype(extractValues), DropValues>) {
filterNulls<int64_t>(rows, false, false);
} else {
readHelper<Reader, velox::common::IsNotNull, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<Reader, velox::common::IsNotNull, isDense>(
filter, rows, extractValues);
}
break;
case velox::common::FilterKind::kBigintRange:
readHelper<Reader, velox::common::BigintRange, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<Reader, velox::common::BigintRange, isDense>(
filter, rows, extractValues);
break;
case velox::common::FilterKind::kNegatedBigintRange:
readHelper<Reader, velox::common::NegatedBigintRange, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<
Reader,
velox::common::NegatedBigintRange,
isDense>(filter, rows, extractValues);
break;
case velox::common::FilterKind::kBigintValuesUsingHashTable:
readHelper<Reader, velox::common::BigintValuesUsingHashTable, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<
Reader,
velox::common::BigintValuesUsingHashTable,
isDense>(filter, rows, extractValues);
break;
case velox::common::FilterKind::kBigintValuesUsingBitmask:
readHelper<Reader, velox::common::BigintValuesUsingBitmask, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<
Reader,
velox::common::BigintValuesUsingBitmask,
isDense>(filter, rows, extractValues);
break;
case velox::common::FilterKind::kNegatedBigintValuesUsingHashTable:
readHelper<
Reader,
velox::common::NegatedBigintValuesUsingHashTable,
isDense>(filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<
Reader,
velox::common::NegatedBigintValuesUsingHashTable,
isDense>(filter, rows, extractValues);
break;
case velox::common::FilterKind::kNegatedBigintValuesUsingBitmask:
readHelper<
Reader,
velox::common::NegatedBigintValuesUsingBitmask,
isDense>(filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<
Reader,
velox::common::NegatedBigintValuesUsingBitmask,
isDense>(filter, rows, extractValues);
break;
default:
readHelper<Reader, velox::common::Filter, isDense>(
filter, rows, extractValues);
static_cast<Reader*>(this)
->template readHelper<Reader, velox::common::Filter, isDense>(
filter, rows, extractValues);
break;
}
}
Expand Down
94 changes: 5 additions & 89 deletions velox/dwio/parquet/reader/TimestampColumnReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,24 +101,23 @@ class TimestampColumnReader : public IntegerColumnReader {
typename TFilter,
bool isDense,
typename ExtractValues>
void readHelperTimestamp(
void readHelper(
velox::common::Filter* filter,
const RowSet& rows,
ExtractValues extractValues) {
if constexpr (std::is_same_v<TFilter, velox::common::TimestampRange>) {
if (auto* range = dynamic_cast<common::TimestampRange*>(filter)) {
// Convert TimestampRange to ParquetInt96TimestampRange.
auto* range = reinterpret_cast<common::TimestampRange*>(filter);
ParquetInt96TimestampRange newRange = ParquetInt96TimestampRange(
range->lower(), range->upper(), range->nullAllowed());
reinterpret_cast<Reader*>(this)->Reader::readWithVisitor(
this->readWithVisitor(
rows,
dwio::common::ColumnVisitor<
int128_t,
ParquetInt96TimestampRange,
ExtractValues,
isDense>(newRange, this, rows, extractValues));
} else {
reinterpret_cast<Reader*>(this)->Reader::readWithVisitor(
this->readWithVisitor(
rows,
dwio::common::
ColumnVisitor<int128_t, TFilter, ExtractValues, isDense>(
Expand All @@ -130,97 +129,14 @@ class TimestampColumnReader : public IntegerColumnReader {
return;
}

template <
typename Reader,
bool isDense,
bool kEncodingHasNulls,
typename ExtractValues>
void processTimestampFilter(
velox::common::Filter* filter,
ExtractValues extractValues,
const RowSet& rows) {
if (filter == nullptr) {
readHelperTimestamp<Reader, velox::common::AlwaysTrue, isDense>(
&dwio::common::alwaysTrue(), rows, extractValues);
return;
}

switch (filter->kind()) {
case velox::common::FilterKind::kAlwaysTrue:
readHelperTimestamp<Reader, velox::common::AlwaysTrue, isDense>(
filter, rows, extractValues);
break;
case velox::common::FilterKind::kIsNull:
if constexpr (kEncodingHasNulls) {
filterNulls<int64_t>(
rows,
true,
!std::
is_same_v<decltype(extractValues), dwio::common::DropValues>);
} else {
readHelperTimestamp<Reader, velox::common::IsNull, isDense>(
filter, rows, extractValues);
}
break;
case velox::common::FilterKind::kIsNotNull:
if constexpr (
kEncodingHasNulls &&
std::is_same_v<decltype(extractValues), dwio::common::DropValues>) {
filterNulls<int64_t>(rows, false, false);
} else {
readHelperTimestamp<Reader, velox::common::IsNotNull, isDense>(
filter, rows, extractValues);
}
break;
case velox::common::FilterKind::kTimestampRange:
readHelperTimestamp<Reader, velox::common::TimestampRange, isDense>(
filter, rows, extractValues);
break;
default:
VELOX_UNREACHABLE();
}
}

template <typename Reader, bool kEncodingHasNulls>
void readTimestamp(const RowSet& rows) {
const bool isDense = rows.back() == rows.size() - 1;
velox::common::Filter* filter =
scanSpec_->filter() ? scanSpec_->filter() : &dwio::common::alwaysTrue();
if (scanSpec_->keepValues()) {
if (scanSpec_->valueHook()) {
if (isDense) {
processValueHook<Reader, true>(rows, scanSpec_->valueHook());
} else {
processValueHook<Reader, false>(rows, scanSpec_->valueHook());
}
} else {
if (isDense) {
processTimestampFilter<Reader, true, kEncodingHasNulls>(
filter, dwio::common::ExtractToReader(this), rows);
} else {
processTimestampFilter<Reader, false, kEncodingHasNulls>(
filter, dwio::common::ExtractToReader(this), rows);
}
}
} else {
if (isDense) {
processTimestampFilter<Reader, true, kEncodingHasNulls>(
filter, dwio::common::DropValues(), rows);
} else {
processTimestampFilter<Reader, false, kEncodingHasNulls>(
filter, dwio::common::DropValues(), rows);
}
}
}

void read(
int64_t offset,
const RowSet& rows,
const uint64_t* /*incomingNulls*/) override {
auto& data = formatData_->as<ParquetData>();
// Use int128_t as a workaround. Timestamp in Velox is of 16-byte length.
prepareRead<int128_t>(offset, rows, nullptr);
readTimestamp<IntegerColumnReader, true>(rows);
readCommon<TimestampColumnReader, true>(rows);
readOffset_ += rows.back() + 1;
}

Expand Down

0 comments on commit 316bf01

Please sign in to comment.