From 043586d26229f63789c6a188536210a8bc4fbf71 Mon Sep 17 00:00:00 2001 From: "Sofiane ALI (contractor)" Date: Tue, 16 Apr 2024 15:38:36 +0200 Subject: [PATCH] Reformat Snowflake files --- .../snowflake/batch/SnowflakeInput.scala | 14 ++--- .../snowflake/batch/SnowflakeOutput.scala | 13 ++-- .../snowflake/batch/SnowflakeInputTest.scala | 12 ++-- .../snowflake/batch/SnowflakeOutputTest.scala | 62 +++++++++---------- .../streaming/SnowflakeOutputTest.scala | 32 +++++----- 5 files changed, 65 insertions(+), 68 deletions(-) diff --git a/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInput.scala b/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInput.scala index 0916c9b..26b1f97 100644 --- a/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInput.scala +++ b/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInput.scala @@ -12,10 +12,10 @@ import org.apache.spark.sql.{DataFrame, SparkSession} * @param config Contains the Typesafe Config object that was used at instantiation to configure this entity. */ case class SnowflakeInput( - options: Map[String, String], - config: Config = ConfigFactory.empty() - ) extends Input - with Logging { + options: Map[String, String], + config: Config = ConfigFactory.empty() +) extends Input + with Logging { val SNOWFLAKE_CONNECTOR_NAME = "net.snowflake.spark.snowflake" @@ -27,7 +27,7 @@ case class SnowflakeInput( * @throws Exception If the exactly one of the dateRange/dateColumn fields is None. */ override def read(implicit spark: SparkSession): DataFrame = { - spark.read.format(SNOWFLAKE_CONNECTOR_NAME).options(options).load() + spark.read.format(SNOWFLAKE_CONNECTOR_NAME).options(options).load() } } @@ -42,6 +42,6 @@ object SnowflakeInput { * @throws com.typesafe.config.ConfigException If any of the mandatory fields is not available in the config argument. */ def apply(implicit config: Config): SnowflakeInput = { - SnowflakeInput(options = getOptions , config = config) + SnowflakeInput(options = getOptions, config = config) } -} \ No newline at end of file +} diff --git a/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutput.scala b/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutput.scala index 248ddaf..b5257ec 100644 --- a/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutput.scala +++ b/core/src/main/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutput.scala @@ -13,15 +13,14 @@ import org.apache.spark.sql.{Dataset, SparkSession} * @param config the config object. */ case class SnowflakeOutput( - mode: String, - options: Map[String, String], - config: Config = ConfigFactory.empty() - ) extends Output - with Logging { + mode: String, + options: Map[String, String], + config: Config = ConfigFactory.empty() +) extends Output + with Logging { val SNOWFLAKE_CONNECTOR_NAME = "net.snowflake.spark.snowflake" - /** * Writes data to this output. * @@ -54,4 +53,4 @@ object SnowflakeOutput { SnowflakeOutput(mode = mode, options = getOptions, config = config) } -} \ No newline at end of file +} diff --git a/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInputTest.scala b/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInputTest.scala index 57aec50..f8d1a93 100644 --- a/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInputTest.scala +++ b/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeInputTest.scala @@ -12,9 +12,9 @@ class SnowflakeInputTest extends AnyWordSpec with Matchers { val config = ConfigFactory.parseMap( Map( "Input" -> Map( - "Name" -> "my-test-snowflake", - "Type" -> "com.amadeus.dataio.pipes.snowflake.batch.SnowflakeInput", - "Options" -> Map( + "Name" -> "my-test-snowflake", + "Type" -> "com.amadeus.dataio.pipes.snowflake.batch.SnowflakeInput", + "Options" -> Map( "sfDatabase" -> "TESTDATABASE", "sfSchema" -> "TESTSCHEMA", "sfUser" -> "TESTUSER", @@ -43,8 +43,8 @@ class SnowflakeInputTest extends AnyWordSpec with Matchers { val config = ConfigFactory.parseMap( Map( "Input" -> Map( - "Name" -> "my-test-snowflake", - "Type" -> "com.amadeus.dataio.pipes.snowflake.batch.SnowflakeInput", + "Name" -> "my-test-snowflake", + "Type" -> "com.amadeus.dataio.pipes.snowflake.batch.SnowflakeInput" ) ) ) @@ -55,4 +55,4 @@ class SnowflakeInputTest extends AnyWordSpec with Matchers { } -} \ No newline at end of file +} diff --git a/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutputTest.scala b/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutputTest.scala index 7e84950..a54a81c 100644 --- a/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutputTest.scala +++ b/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/batch/SnowflakeOutputTest.scala @@ -5,7 +5,6 @@ import com.typesafe.config.{ConfigException, ConfigFactory} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec - class SnowflakeOutputTest extends AnyWordSpec with Matchers { "SnowflakeOutput" should { @@ -14,16 +13,16 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { val config = ConfigFactory.parseMap( Map( "Output" -> Map( - "Type" -> "com.amadeus.dataio.output.streaming.SnowflakeOutput", - "Name" -> "my-test-snowflake", - "Mode" -> "append", - "Options" -> Map( - "dbTable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", - "sfWarehouse" -> "warehouse", + "Type" -> "com.amadeus.dataio.output.streaming.SnowflakeOutput", + "Name" -> "my-test-snowflake", + "Mode" -> "append", + "Options" -> Map( + "dbTable" -> "test-table", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", + "sfWarehouse" -> "warehouse", "column_mapping" -> "name" ) ) @@ -33,12 +32,12 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { val snowflakeStreamOutput = SnowflakeOutput.apply(config.getConfig("Output")) val expectedSnowflakeOptions = Map( - "dbTable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", - "sfWarehouse" -> "warehouse", + "dbTable" -> "test-table", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", + "sfWarehouse" -> "warehouse", "column_mapping" -> "name" ) @@ -50,16 +49,16 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { val config = ConfigFactory.parseMap( Map( - "Output" -> Map( - "Type" -> "com.amadeus.dataio.output.streaming.SnowflakeOutput", - "Name" -> "my-test-snowflake", - "Options" -> Map( - "dbTable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", - "sfWarehouse" -> "warehouse", + "Output" -> Map( + "Type" -> "com.amadeus.dataio.output.streaming.SnowflakeOutput", + "Name" -> "my-test-snowflake", + "Options" -> Map( + "dbTable" -> "test-table", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", + "sfWarehouse" -> "warehouse", "column_mapping" -> "name" ) ) @@ -74,9 +73,9 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { val config = ConfigFactory.parseMap( Map( - "Output" -> Map( - "Type" -> "com.amadeus.dataio.output.streaming.SnowflakeOutput", - "Name" -> "my-test-snowflake", + "Output" -> Map( + "Type" -> "com.amadeus.dataio.output.streaming.SnowflakeOutput", + "Name" -> "my-test-snowflake" ) ) ) @@ -85,7 +84,6 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { intercept[ConfigException](snowflakeStreamOutput) } - } -} \ No newline at end of file +} diff --git a/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/streaming/SnowflakeOutputTest.scala b/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/streaming/SnowflakeOutputTest.scala index 685f8d1..7448b85 100644 --- a/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/streaming/SnowflakeOutputTest.scala +++ b/core/src/test/scala/com/amadeus/dataio/pipes/snowflake/streaming/SnowflakeOutputTest.scala @@ -23,10 +23,10 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { "Timeout" -> "24", "Options" -> Map( "dbtable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", "sfWarehouse" -> "test-warehouse", "sfRole" -> "tester" ) @@ -67,10 +67,10 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { "AddTimestampOnInsert" -> true, "Options" -> Map( "dbtable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", "sfWarehouse" -> "test-warehouse", "sfRole" -> "tester" ) @@ -111,10 +111,10 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { "AddTimestampOnInsert" -> false, "Options" -> Map( "dbtable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", "sfWarehouse" -> "test-warehouse", "sfRole" -> "tester" ) @@ -126,10 +126,10 @@ class SnowflakeOutputTest extends AnyWordSpec with Matchers { val expectedSnowflakeOptions = Map( "dbtable" -> "test-table", - "sfUrl" -> "http://snowflake.com", - "sfUser" -> "my-user", - "sfDatabase" -> "db", - "sfSchema" -> "test-schema", + "sfUrl" -> "http://snowflake.com", + "sfUser" -> "my-user", + "sfDatabase" -> "db", + "sfSchema" -> "test-schema", "sfWarehouse" -> "test-warehouse", "sfRole" -> "tester" )