From fb113d6258d40aac2886993a3e0711919ad31779 Mon Sep 17 00:00:00 2001 From: hezhao2 Date: Sun, 7 Apr 2024 12:18:39 +0800 Subject: [PATCH] disable the rule that checks the maxPartitions for dsv2 --- .../scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala | 3 +-- .../test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala index 871c21fba58..14e0eb08c0f 100644 --- a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala @@ -240,8 +240,7 @@ case class MaxScanStrategy(session: SparkSession) _, relation @ DataSourceV2ScanRelation(_, _, _, _, _)) => val table = relation.relation.table - if (table.partitioning().nonEmpty && - relation.scan.isInstanceOf[SupportsReportPartitioning]) { + if (table.partitioning().nonEmpty) { val partitionColumnNames = table.partitioning().map(_.describe()) val stats = relation.computeStats() lazy val scanFileSize = stats.sizeInBytes diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala index dc009caf32c..954a2e4296a 100644 --- a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala @@ -608,7 +608,7 @@ trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest { assert(e.getMessage == "Script transformation is not allowed") } } - + test("watchdog with scan maxFileSize -- data source v2") { val df = spark.read.format(classOf[ReportStatisticsAndPartitionAwareDataSource].getName).load() df.createOrReplaceTempView("test")