feat: Allow non standard partition functions in ScaleWriterPartitioni…

…ngLocalPartition (facebookincubator#11762) Summary: Pull Request resolved: facebookincubator#11762 For example Hive connector partitioning function Reviewed By: xiaoxmeng, Yuhta, tanjialiang Differential Revision: D66833831 fbshipit-source-id: 9ee130ec5e90d2d31973ef14344cd769da63d06a
bikramSingh91 · Dec 9, 2024 · b9cce6d · b9cce6d
1 parent 9a31f1c
commit b9cce6d
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 14 deletions.
diff --git a/velox/exec/LocalPlanner.cpp b/velox/exec/LocalPlanner.cpp
@@ -76,15 +76,13 @@ std::unique_ptr<Operator> createScaleWriterLocalPartition(
     const std::shared_ptr<const core::LocalPartitionNode>& localPartitionNode,
     int32_t operatorId,
     DriverCtx* ctx) {
-  if (dynamic_cast<const HashPartitionFunctionSpec*>(
+  if (dynamic_cast<const RoundRobinPartitionFunctionSpec*>(
           &localPartitionNode->partitionFunctionSpec())) {
-    return std::make_unique<ScaleWriterPartitioningLocalPartition>(
+    return std::make_unique<ScaleWriterLocalPartition>(
         operatorId, ctx, localPartitionNode);
   }
 
-  VELOX_CHECK_NOT_NULL(dynamic_cast<const RoundRobinPartitionFunctionSpec*>(
-      &localPartitionNode->partitionFunctionSpec()));
-  return std::make_unique<ScaleWriterLocalPartition>(
+  return std::make_unique<ScaleWriterPartitioningLocalPartition>(
       operatorId, ctx, localPartitionNode);
 }
 

diff --git a/velox/exec/ScaleWriterLocalPartition.cpp b/velox/exec/ScaleWriterLocalPartition.cpp
@@ -57,10 +57,6 @@ ScaleWriterPartitioningLocalPartition::ScaleWriterPartitioningLocalPartition(
       : planNode->partitionFunctionSpec().create(
             numTablePartitions_,
             /*localExchange=*/true);
-  if (partitionFunction_ != nullptr) {
-    VELOX_CHECK_NOT_NULL(
-        dynamic_cast<HashPartitionFunction*>(partitionFunction_.get()));
-  }
 }
 
 void ScaleWriterPartitioningLocalPartition::initialize() {

diff --git a/velox/exec/tests/PlanNodeSerdeTest.cpp b/velox/exec/tests/PlanNodeSerdeTest.cpp
@@ -43,6 +43,7 @@ class PlanNodeSerdeTest : public testing::Test,
     connector::hive::LocationHandle::registerSerDe();
     connector::hive::HiveColumnHandle::registerSerDe();
     connector::hive::HiveInsertTableHandle::registerSerDe();
+    connector::hive::registerHivePartitionFunctionSerDe();
     core::PlanNode::registerSerDe();
     core::ITypedExpr::registerSerDe();
     registerPartitionFunctionSerDe();

diff --git a/velox/exec/tests/utils/PlanBuilder.cpp b/velox/exec/tests/utils/PlanBuilder.cpp
@@ -1293,12 +1293,20 @@ PlanBuilder& PlanBuilder::localPartition(const std::vector<std::string>& keys) {
 
 PlanBuilder& PlanBuilder::scaleWriterlocalPartition(
     const std::vector<std::string>& keys) {
-  planNode_ = createLocalPartitionNode(
+  std::vector<column_index_t> keyIndices;
+  keyIndices.reserve(keys.size());
+  for (const auto& key : keys) {
+    keyIndices.push_back(planNode_->outputType()->getChildIdx(key));
+  }
+  auto hivePartitionFunctionFactory =
+      std::make_shared<HivePartitionFunctionSpec>(
+          1009, keyIndices, std::vector<VectorPtr>{});
+  planNode_ = std::make_shared<core::LocalPartitionNode>(
       nextPlanNodeId(),
-      exprs(keys, planNode_->outputType()),
-      /*scaleWriter=*/true,
-      {planNode_},
-      pool_);
+      core::LocalPartitionNode::Type::kRepartition,
+      true,
+      hivePartitionFunctionFactory,
+      std::vector{planNode_});
   return *this;
 }