wip

microsoft · Jul 19, 2024 · e8ee16c · e8ee16c
1 parent ced6547
commit e8ee16c
Show file tree

Hide file tree

Showing 12 changed files with 27 additions and 12 deletions.
diff --git a/cognitive/src/test/python/synapsemltest/services/test_simple.py b/cognitive/src/test/python/synapsemltest/services/test_simple.py
@@ -15,6 +15,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class SimpleHTTPTransformerSmokeTest(unittest.TestCase):
     def test_simple(self):
         df = spark.createDataFrame([("foo",) for x in range(20)], ["data"]).withColumn(

diff --git a/core/src/main/python/synapse/ml/core/init_spark.py b/core/src/main/python/synapse/ml/core/init_spark.py
@@ -3,18 +3,22 @@
 
 from synapse.ml.core import __spark_package_version__
 
+
 def init_spark():
     from pyspark.sql import SparkSession, SQLContext
 
-    return (SparkSession.builder
-            .master("local[*]")
-            .appName("PysparkTests")
-            .config("spark.jars.packages",
-                    "com.microsoft.azure:synapseml_2.12:" +
-                    __spark_package_version__ +
-                    ",org.apache.spark:spark-avro_2.12:3.4.1")
-            .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
-            .config("spark.executor.heartbeatInterval", "60s")
-            .config("spark.sql.shuffle.partitions", 10)
-            .config("spark.sql.crossJoin.enabled", "true")
-            .getOrCreate())
+    return (
+        SparkSession.builder.master("local[*]")
+        .appName("PysparkTests")
+        .config(
+            "spark.jars.packages",
+            "com.microsoft.azure:synapseml_2.12:"
+            + __spark_package_version__
+            + ",org.apache.spark:spark-avro_2.12:3.4.1",
+        )
+        .config("spark.jars.repositories", "https://mmlspark.azureedge.net/maven")
+        .config("spark.executor.heartbeatInterval", "60s")
+        .config("spark.sql.shuffle.partitions", 10)
+        .config("spark.sql.crossJoin.enabled", "true")
+        .getOrCreate()
+    )
diff --git a/core/src/test/python/synapsemltest/core/test_logging.py b/core/src/test/python/synapsemltest/core/test_logging.py
@@ -12,6 +12,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class SampleTransformer(SynapseMLLogger):
     def __init__(self):
         super().__init__(log_level=logging.DEBUG)

diff --git a/core/src/test/python/synapsemltest/core/test_template.py b/core/src/test/python/synapsemltest/core/test_template.py
@@ -12,6 +12,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class TemplateSpec(unittest.TestCase):
     def create_sample_dataframe(self):
         schema = t.StructType(

diff --git a/core/src/test/python/synapsemltest/cyber/anamoly/test_complement_access.py b/core/src/test/python/synapsemltest/cyber/anamoly/test_complement_access.py
@@ -12,6 +12,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class TestComplementAccessTransformer(unittest.TestCase):
     def create_dataframe(self) -> DataFrame:
         schema = t.StructType(

diff --git a/core/src/test/python/synapsemltest/cyber/explain_tester.py b/core/src/test/python/synapsemltest/cyber/explain_tester.py
@@ -9,6 +9,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class ExplainTester:
     def check_explain(
         self,

diff --git a/core/src/test/python/synapsemltest/cyber/feature/test_indexers.py b/core/src/test/python/synapsemltest/cyber/feature/test_indexers.py
@@ -12,6 +12,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class TestIndexers(unittest.TestCase):
     def create_sample_dataframe(self):
         schema = t.StructType(

diff --git a/core/src/test/python/synapsemltest/cyber/feature/test_scalers.py b/core/src/test/python/synapsemltest/cyber/feature/test_scalers.py
@@ -12,6 +12,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class TestScalers(unittest.TestCase):
     def create_sample_dataframe(self):
         schema = t.StructType(

diff --git a/core/src/test/python/synapsemltest/cyber/utils/test_spark_utils.py b/core/src/test/python/synapsemltest/cyber/utils/test_spark_utils.py
@@ -20,6 +20,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class TestDataFrameUtils(unittest.TestCase):
     def create_sample_dataframe(self):
         dataframe = sc.createDataFrame(

diff --git a/core/src/test/python/synapsemltest/nn/test_ball_tree.py b/core/src/test/python/synapsemltest/nn/test_ball_tree.py
@@ -11,6 +11,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class NNSpec(unittest.TestCase):
     def test_bindings(self):
         cbt = ConditionalBallTree([[1.0, 2.0], [2.0, 3.0]], [1, 2], ["foo", "bar"], 50)

diff --git a/vw/src/test/python/synapsemltest/vw/test_vw.py b/vw/src/test/python/synapsemltest/vw/test_vw.py
@@ -15,6 +15,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 class VowpalWabbitSpec(unittest.TestCase):
     def get_data(self):
         # create sample data

diff --git a/vw/src/test/python/synapsemltest/vw/test_vw_cb.py b/vw/src/test/python/synapsemltest/vw/test_vw_cb.py
@@ -20,6 +20,7 @@
 spark = init_spark()
 sc = SQLContext(spark.sparkContext)
 
+
 def has_column(df, col):
     try:
         df[col]
-Original file line number
+Diff line change
@@ Expand Up / @@ -20,6 +20,7 @@ @@
     spark = init_spark()
     sc = SQLContext(spark.sparkContext)
     def has_column(df, col):
         try:
             df[col]
@@ Expand Down @@