diff --git a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala index 64980209c59e..e7d2b22255f7 100644 --- a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala +++ b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.catalyst.{AggregateFunctionRewriteRule, FlushableHas import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Attribute, Cast, CreateNamedStruct, ElementAt, Expression, ExpressionInfo, Generator, GetArrayItem, GetMapValue, GetStructField, If, IsNaN, Literal, Murmur3Hash, NamedExpression, NaNvl, PosExplode, Round, SortOrder, StringSplit, StringTrim} +import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Attribute, Cast, CreateNamedStruct, ElementAt, Expression, ExpressionInfo, Generator, GetArrayItem, GetMapValue, GetStructField, If, IsNaN, Literal, Murmur3Hash, NamedExpression, NaNvl, PosExplode, Round, SortOrder, StringSplit, StringTrim, Uuid} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, HLLAdapter} import org.apache.spark.sql.catalyst.optimizer.BuildSide import org.apache.spark.sql.catalyst.plans.JoinType @@ -129,6 +129,16 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi { ) } + /** Transform Uuid to Substrait. */ + override def genUuidTransformer( + substraitExprName: String, + original: Uuid): ExpressionTransformer = { + GenericExpressionTransformer( + substraitExprName, + Seq(LiteralTransformer(Literal(original.randomSeed.get))), + original) + } + /** Transform map_entries to Substrait. */ override def genMapEntriesTransformer( substraitExprName: String, diff --git a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala index 6201a372df64..0ff2bd0d705d 100644 --- a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala @@ -27,7 +27,7 @@ import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat.{DwrfReadFormat, OrcReadFormat, ParquetReadFormat} import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, Expression, Lag, Lead, Literal, NamedExpression, NthValue, NTile, PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder, SpecialFrameBoundary, SpecifiedWindowFrame} +import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, Expression, Lag, Lead, Literal, NamedExpression, NthValue, NTile, PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Sum} import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.util.CharVarcharUtils @@ -388,6 +388,7 @@ object BackendSettings extends BackendSettingsApi { // Block directly falling back the below functions by FallbackEmptySchemaRelation. case alias: Alias => checkExpr(alias.child) case _: Rand => true + case _: Uuid => true case _ => false } } diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala index e5bee97611a2..f7f1dc90e885 100644 --- a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala @@ -513,6 +513,12 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite { } } + test("Test uuid function") { + runQueryAndCompare("""SELECT uuid() from lineitem limit 100""".stripMargin, false) { + checkOperatorMatch[ProjectExecTransformer] + } + } + test("regexp_replace") { runQueryAndCompare( "SELECT regexp_replace(l_partkey, '\\w', 'something') FROM lineitem limit 100") { diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index e9ba9d1f2b93..56cb95c1bb11 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -428,3 +428,4 @@ Gluten supports 199 functions. (Draw to right to see all data types) | spark_partition_id | | | S | | | | | | | | | | | | | | | | | | | | | stack | | | | | | | | | | | | | | | | | | | | | | | | xxhash64 | xxhash64 | xxhash64 | | | | | | | | | | | | | | | | | | | | | +| uuid | uuid | uuid | S | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala index 4f40a8a238b4..4166c4a68863 100644 --- a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala @@ -192,6 +192,10 @@ trait SparkPlanExecApi { throw new GlutenNotSupportException("NaNvl is not supported") } + def genUuidTransformer(substraitExprName: String, original: Uuid): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(), original) + } + /** Transform map_entries to Substrait. */ def genMapEntriesTransformer( substraitExprName: String, diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala index ccd31714703a..aad06bfb211a 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala @@ -537,6 +537,8 @@ object ExpressionConverter extends SQLConfHelper with Logging { val childrenTransformers = e.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) e.getTransformer(childrenTransformers) + case u: Uuid => + BackendsApiManager.getSparkPlanExecApiInstance.genUuidTransformer(substraitExprName, u) case expr => GenericExpressionTransformer( substraitExprName,