Support concat function

apache · Mar 24, 2024 · 4653a1e · 4653a1e
1 parent dafb371
commit 4653a1e
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 4 deletions.
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
@@ -179,6 +179,7 @@ object CHExpressionUtil {
     SPARK_PARTITION_ID -> DefaultValidator(),
     URL_DECODE -> DefaultValidator(),
     SKEWNESS -> DefaultValidator(),
-    BIT_LENGTH -> DefaultValidator()
+    BIT_LENGTH -> DefaultValidator(),
+    MAP_CONCAT -> DefaultValidator()
   )
 }
diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
@@ -567,4 +567,14 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite {
     }
   }
 
+  test("Test map_concat function") {
+    runQueryAndCompare("select map_concat(map(4, 6), map(7, 8))") {
+      checkOperatorMatch[ProjectExecTransformer]
+    }
+    runQueryAndCompare(
+      "select map_concat(map(l_returnflag, l_comment), map(l_shipmode, l_comment)) " +
+        "from lineitem limit 1") {
+      checkOperatorMatch[ProjectExecTransformer]
+    }
+  }
 }
diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md
@@ -276,7 +276,7 @@ Gluten supports 199 functions. (Draw to right to see all data types)
 | filter                        | filter                 | filter                |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | flatten                       | flatten                |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | map                           | map                    | map                   | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
-| map_concat                    | map_concat             |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
+| map_concat                    | map_concat             | map_concat            | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | map_entries                   | map_entries            |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | map_filter                    | map_filter             | map_filter            |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | get_map_value                 |                        | element_at            | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       | S    |        |      |
@@ -416,7 +416,7 @@ Gluten supports 199 functions. (Draw to right to see all data types)
 | java_method                   |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | least                         | least                  | least                 | S      |                        |         |      |       |     | S    | S     | S      | S    | S         |        |         |      |        |          |       |      |        |      |
 | md5                           | md5                    |                       | S      |                        |         | S    |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
-| monotonically_increasing_id   |                        |                       |   S     |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
+| monotonically_increasing_id   |                        |                       | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | nanvl                         |                        |                       | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | nvl                           |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | nvl2                          |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
@@ -428,4 +428,4 @@ Gluten supports 199 functions. (Draw to right to see all data types)
 | spark_partition_id            |                        |                       | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | stack                         |                        |                       |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
 | xxhash64                      | xxhash64               | xxhash64              |        |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
-| uuid                      | uuid               | uuid              | S       |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
+| uuid                      | uuid               | uuid                  | S      |                        |         |      |       |     |      |       |        |      |           |        |         |      |        |          |       |      |        |      |
diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
@@ -226,6 +226,7 @@ object ExpressionMappings {
     Sig[MapFromArrays](MAP_FROM_ARRAYS),
     Sig[MapEntries](MAP_ENTRIES),
     Sig[StringToMap](STR_TO_MAP),
+    Sig[MapConcat](MAP_CONCAT),
     // Struct functions
     Sig[GetStructField](GET_STRUCT_FIELD),
     Sig[CreateNamedStruct](NAMED_STRUCT),

diff --git a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
@@ -237,6 +237,7 @@ object ExpressionNames {
   final val MAP_FROM_ARRAYS = "map_from_arrays"
   final val MAP_ENTRIES = "map_entries"
   final val STR_TO_MAP = "str_to_map"
+  final val MAP_CONCAT = "map_concat"
 
   // struct functions
   final val GET_STRUCT_FIELD = "get_struct_field"