Skip to content

Commit

Permalink
Optimize MergedFlatMapFieldReader 3 times faster
Browse files Browse the repository at this point in the history
Summary:
X-link: facebookincubator/nimble#24

1. Copy map values in batch instead of row by row
2. Optimize CPU cache access pattern to avoid thrashing

Reviewed By: xiaoxmeng

Differential Revision: D54959863
  • Loading branch information
Yuhta authored and facebook-github-bot committed Mar 18, 2024
1 parent 5cfb8eb commit f70f5b3
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 13 deletions.
11 changes: 6 additions & 5 deletions velox/dwio/common/FlatMapHelper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ void reset(VectorPtr& vector, vector_size_t size, bool hasNulls) {

void initializeStringVector(
VectorPtr& vector,
const TypePtr& type,
memory::MemoryPool& pool,
const std::vector<const BaseVector*>& vectors) {
vector_size_t size = 0;
Expand All @@ -64,7 +65,7 @@ void initializeStringVector(
}
}
initializeFlatVector<StringView>(
vector, pool, VARCHAR(), size, hasNulls, std::move(buffers));
vector, pool, type, size, hasNulls, std::move(buffers));
}

} // namespace detail
Expand Down Expand Up @@ -103,19 +104,19 @@ void initializeVectorImpl(
template <>
void initializeVectorImpl<TypeKind::VARCHAR>(
VectorPtr& vector,
const TypePtr& /* type */,
const TypePtr& type,
memory::MemoryPool& pool,
const std::vector<const BaseVector*>& vectors) {
detail::initializeStringVector(vector, pool, vectors);
detail::initializeStringVector(vector, type, pool, vectors);
}

template <>
void initializeVectorImpl<TypeKind::VARBINARY>(
VectorPtr& vector,
const TypePtr& /* type */,
const TypePtr& type,
memory::MemoryPool& pool,
const std::vector<const BaseVector*>& vectors) {
detail::initializeStringVector(vector, pool, vectors);
detail::initializeStringVector(vector, type, pool, vectors);
}

namespace {
Expand Down
8 changes: 0 additions & 8 deletions velox/dwio/common/FlatMapHelper.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,6 @@ void resetIfNotWritable(VectorPtr& vector, const T&... buffer) {
FOLLY_POP_WARNING
}

// Initialize string vector.
void initializeStringVector(
VectorPtr& vector,
memory::MemoryPool& pool,
vector_size_t size,
bool hasNulls,
std::vector<BufferPtr>&& stringBuffers);

} // namespace detail

// Struct for keeping track flatmap key stream metrics.
Expand Down

0 comments on commit f70f5b3

Please sign in to comment.