From 17924b81703aaabd4855c26a7c71e69eec52cd16 Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Thu, 21 Mar 2024 03:38:01 +0000 Subject: [PATCH 1/2] [VL] Daily Update Velox Version (2024_03_21) Signed-off-by: glutenperfbot --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index f9f6385743fd..09fdeb70cbb0 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_03_20 +VELOX_BRANCH=2024_03_21 VELOX_HOME="" #Set on run gluten on HDFS From 6177150b025dae70ae35d70fddc1d973bc388563 Mon Sep 17 00:00:00 2001 From: yan ma Date: Fri, 22 Mar 2024 01:06:45 +0800 Subject: [PATCH 2/2] remove configs for threshold based spilling --- cpp/velox/compute/WholeStageResultIterator.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 9c4367b721e0..8d106950d1a3 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -42,10 +42,6 @@ const std::string kSpillStrategyDefaultValue = "auto"; const std::string kAggregationSpillEnabled = "spark.gluten.sql.columnar.backend.velox.aggregationSpillEnabled"; const std::string kJoinSpillEnabled = "spark.gluten.sql.columnar.backend.velox.joinSpillEnabled"; const std::string kOrderBySpillEnabled = "spark.gluten.sql.columnar.backend.velox.orderBySpillEnabled"; -const std::string kAggregationSpillMemoryThreshold = - "spark.gluten.sql.columnar.backend.velox.aggregationSpillMemoryThreshold"; -const std::string kJoinSpillMemoryThreshold = "spark.gluten.sql.columnar.backend.velox.joinSpillMemoryThreshold"; -const std::string kOrderBySpillMemoryThreshold = "spark.gluten.sql.columnar.backend.velox.orderBySpillMemoryThreshold"; const std::string kMaxSpillLevel = "spark.gluten.sql.columnar.backend.velox.maxSpillLevel"; const std::string kMaxSpillFileSize = "spark.gluten.sql.columnar.backend.velox.maxSpillFileSize"; const std::string kMinSpillRunSize = "spark.gluten.sql.columnar.backend.velox.minSpillRunSize"; @@ -507,12 +503,6 @@ std::unordered_map WholeStageResultIterator::getQueryC std::to_string(veloxCfg_->get(kJoinSpillEnabled, true)); configs[velox::core::QueryConfig::kOrderBySpillEnabled] = std::to_string(veloxCfg_->get(kOrderBySpillEnabled, true)); - configs[velox::core::QueryConfig::kAggregationSpillMemoryThreshold] = std::to_string( - veloxCfg_->get(kAggregationSpillMemoryThreshold, 0)); // spill only when input doesn't fit - configs[velox::core::QueryConfig::kJoinSpillMemoryThreshold] = - std::to_string(veloxCfg_->get(kJoinSpillMemoryThreshold, 0)); // spill only when input doesn't fit - configs[velox::core::QueryConfig::kOrderBySpillMemoryThreshold] = - std::to_string(veloxCfg_->get(kOrderBySpillMemoryThreshold, 0)); // spill only when input doesn't fit configs[velox::core::QueryConfig::kMaxSpillLevel] = std::to_string(veloxCfg_->get(kMaxSpillLevel, 4)); configs[velox::core::QueryConfig::kMaxSpillFileSize] = std::to_string(veloxCfg_->get(kMaxSpillFileSize, 20L * 1024 * 1024));