Fix assert not work on DF with single column

mrpowers-io · Dec 28, 2024 · e6caa63 · e6caa63
1 parent cd173ef
commit e6caa63
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/core/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetComparer.scala b/core/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetComparer.scala
@@ -194,7 +194,10 @@ Expected DataFrame Row Count: '$expectedCount'
       if (actualCount != expectedCount) {
         throw DatasetCountMismatch(countMismatchMessage(actualCount, expectedCount))
       }
-
+      val joined = ds1
+        .joinPair(ds2, primaryKeys)
+      joined.show(false)
+      joined.printSchema()
       val unequalDf = ds1
         .joinPair(ds2, primaryKeys)
         .filter((p: (T, T)) => !customEquals(p._1, p._2))

diff --git a/core/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetUtils.scala b/core/src/main/scala/com/github/mrpowers/spark/fast/tests/DatasetUtils.scala
@@ -2,7 +2,7 @@ package com.github.mrpowers.spark.fast.tests
 
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions._
-import org.apache.spark.sql.{DataFrame, Dataset, Encoder}
+import org.apache.spark.sql.{DataFrame, Dataset, Encoder, Row}
 
 import scala.reflect.ClassTag
 
@@ -33,7 +33,7 @@ private object DatasetUtils {
         val leftCols            = columns.map(n => col(s"l.$n"))
         val rightCols           = columns.map(n => col(s"r.$n"))
         val (pair1, pair2) =
-          if (columns.length == 1)
+          if (columns.length == 1 && !(implicitly[ClassTag[T]].runtimeClass == classOf[Row]))
             (leftCols.head, rightCols.head)
           else
             (struct(leftCols: _*), struct(rightCols: _*))