Skip to content

Commit

Permalink
Make metadata test work on spark 3.0/3.1/3.2
Browse files Browse the repository at this point in the history
  • Loading branch information
@zeotuan
zeotuan committed Oct 12, 2024
1 parent 1791243 commit 9aae738
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 20 deletions.
25 changes: 13 additions & 12 deletions core/src/test/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparerTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import org.apache.spark.sql.types.{DoubleType, IntegerType, MetadataBuilder, Str
import SparkSessionExt._
import com.github.mrpowers.spark.fast.tests.SchemaComparer.DatasetSchemaMismatch
import com.github.mrpowers.spark.fast.tests.StringExt.StringOps
import org.apache.spark.sql.functions.col
import org.scalatest.freespec.AnyFreeSpec

class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with SparkSessionTestWrapper {
Expand Down Expand Up @@ -320,7 +321,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small int").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small int").build()))

val expectedDF = spark
.createDF(
Expand All @@ -330,7 +331,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small number").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small number").build()))

assertLargeDataFrameEquality(sourceDF, expectedDF)
}
Expand All @@ -344,7 +345,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small int").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small int").build()))

val expectedDF = spark
.createDF(
Expand All @@ -354,7 +355,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small number").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small number").build()))

intercept[DatasetSchemaMismatch] {
assertLargeDataFrameEquality(sourceDF, expectedDF, ignoreMetadata = false)
Expand Down Expand Up @@ -517,7 +518,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small int").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small int").build()))

val expectedDF = spark
.createDF(
Expand All @@ -527,7 +528,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small number").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small number").build()))

assertApproximateDataFrameEquality(sourceDF, expectedDF, precision = 0.0000001)
}
Expand All @@ -541,7 +542,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small int").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small int").build()))

val expectedDF = spark
.createDF(
Expand All @@ -551,7 +552,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small number").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small number").build()))

intercept[DatasetSchemaMismatch] {
assertApproximateDataFrameEquality(sourceDF, expectedDF, precision = 0.0000001, ignoreMetadata = false)
Expand Down Expand Up @@ -714,7 +715,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small int").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small int").build()))

val expectedDF = spark
.createDF(
Expand All @@ -724,7 +725,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small number").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small number").build()))

assertApproximateSmallDataFrameEquality(sourceDF, expectedDF, precision = 0.0000001)
}
Expand All @@ -738,7 +739,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small int").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small int").build()))

val expectedDF = spark
.createDF(
Expand All @@ -748,7 +749,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
),
List(("number", IntegerType, true))
)
.withMetadata("number", new MetadataBuilder().putString("description", "small number").build())
.withColumn("number", col("number").as("number", new MetadataBuilder().putString("description", "small number").build()))

intercept[DatasetSchemaMismatch] {
assertApproximateSmallDataFrameEquality(sourceDF, expectedDF, precision = 0.0000001, ignoreMetadata = false)
Expand Down
17 changes: 9 additions & 8 deletions core/src/test/scala/com/github/mrpowers/spark/fast/tests/DatasetComparerTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import org.apache.spark.sql.types._
import SparkSessionExt._
import com.github.mrpowers.spark.fast.tests.SchemaComparer.DatasetSchemaMismatch
import com.github.mrpowers.spark.fast.tests.StringExt.StringOps
import org.apache.spark.sql.functions.col
import org.scalatest.freespec.AnyFreeSpec

object Person {
Expand Down Expand Up @@ -415,7 +416,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "name of the person").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the person").build()))
.as[Person]

val ds2 = Seq(
Expand All @@ -424,7 +425,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "name of the individual").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the individual").build()))
.as[Person]

assertLargeDatasetEquality(ds2, ds1)
Expand All @@ -437,7 +438,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "name of the person").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the person").build()))
.as[Person]

val ds2 = Seq(
Expand All @@ -446,7 +447,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "name of the individual").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the individual").build()))
.as[Person]

intercept[DatasetSchemaMismatch] {
Expand Down Expand Up @@ -623,7 +624,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "first name of a person").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the person").build()))
.as[Person]

val ds2 = Seq(
Expand All @@ -632,7 +633,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "first name of an individual").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the individual").build()))
.as[Person]

assertSmallDatasetEquality(ds2, ds1)
Expand All @@ -645,7 +646,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "name of the person").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the person").build()))
.as[Person]

val ds2 = Seq(
Expand All @@ -654,7 +655,7 @@ class DatasetComparerTest extends AnyFreeSpec with DatasetComparer with SparkSes
Person("li", 49),
Person("alice", 5)
).toDS
.withMetadata("name", new MetadataBuilder().putString("description", "name of the individual").build())
.withColumn("name", col("name").as("name", new MetadataBuilder().putString("description", "name of the individual").build()))
.as[Person]

intercept[DatasetSchemaMismatch] {
Expand Down

0 comments on commit 9aae738

Please sign in to comment.