From 96fc4b9bbdaab520bda5616bc6d6149fa08e2934 Mon Sep 17 00:00:00 2001 From: Haojian Zhuang Date: Mon, 19 Nov 2018 13:28:53 +0800 Subject: [PATCH] sparkbench: avoid to verify file format in SparkFuncs.scala commit 992a278f0e2fea6a4333bfe1d7ed98aa95011e0d Author: Emily Curtin Date: Fri Feb 2 14:46:33 2018 -0500 Addressing code review comments Moves format check to front. This commit add verifyFormatOrThrow(outputDir, fileFormat) in verifyOutput(). It causes graph data generator workload failure since the workload only generate output with txt file format. But sparkbench only supports csv, parquet, etc. The confliction of file format causes the workload failure. Now remove the verifyFormatOrThrow() instead. Signed-off-by: Haojian Zhuang --- .../main/scala/com/ibm/sparktc/sparkbench/utils/SparkFuncs.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/src/main/scala/com/ibm/sparktc/sparkbench/utils/SparkFuncs.scala b/utils/src/main/scala/com/ibm/sparktc/sparkbench/utils/SparkFuncs.scala index 1b20947a..2667be2a 100644 --- a/utils/src/main/scala/com/ibm/sparktc/sparkbench/utils/SparkFuncs.scala +++ b/utils/src/main/scala/com/ibm/sparktc/sparkbench/utils/SparkFuncs.scala @@ -32,7 +32,6 @@ object SparkFuncs { def verifyOutput(outputDir: Option[String], saveMode: String, spark: SparkSession, fileFormat: Option[String] = None): Unit = { verifyCanWriteOrThrow(outputDir, saveMode, spark, fileFormat) - verifyFormatOrThrow(outputDir, fileFormat) } def verifyCanWrite(outputDir: String, saveMode: String, spark: SparkSession, fileFormat: Option[String] = None): Boolean = {