added singleton SparkSessionProvider #932
Conversation
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
|
|
||
| public class TestDocumenter { | ||
|
|
||
| public static final Log LOG = LogFactory.getLog(TestDocumenter.class); |
Check warning
Code scanning / PMD
Logger should be defined private static final and have the correct class Warning test
| try { | ||
| args = argsUtil.createArgumentsFromJSON(getClass().getResource("/testDocumenter/config.json").getFile()); | ||
| ArgumentsUtil argsUtil = new ArgumentsUtil(); | ||
| IArguments args = argsUtil.createArgumentsFromJSON(getClass().getResource("/testDocumenter/config.json").getFile()); |
Check warning
Code scanning / PMD
Avoid unused local variables such as 'obj'. Warning test
spark/core/src/test/java/zingg/spark/core/recommender/TestStopWordsRecommender.java
Fixed
Show fixed
Hide fixed
| private JavaSparkContext javaSparkContext; | ||
| private ZinggSparkContext zinggSparkContext; | ||
| private IArguments args; | ||
| public static final Log LOG = LogFactory.getLog(SparkSessionProvider.class); |
Check warning
Code scanning / PMD
Logger should be defined private static final and have the correct class Warning test
| zinggSparkContext.init(sparkSession); | ||
| } catch (Throwable e) { | ||
| if (LOG.isDebugEnabled()) | ||
| e.printStackTrace(); |
Check warning
Code scanning / PMD
This statement should have braces Warning test
| if (sparkSessionProvider == null) { | ||
| sparkSessionProvider = new SparkSessionProvider(); | ||
| sparkSessionProvider.initializeSession(); | ||
| } |
Check warning
Code scanning / PMD
Singleton is not thread safe Warning test
| @@ -1,11 +1,11 @@ | |||
| package zingg.common.core.sparkFrame; | |||
| package zingg.spark.core.sparkFrame; | |||
Check warning
Code scanning / PMD
Package name contains upper case characters Warning test
| import java.util.List; | ||
| import java.util.stream.IntStream; | ||
|
|
||
| public class DataFrameUtility { |
Check warning
Code scanning / PMD
This utility class has a non-private constructor Warning test
| for (int n = 0; n < numCols; ++n) { | ||
| structType = structType.add("col" + n, DataTypes.DoubleType, false); | ||
| rowValues.add(0d); | ||
| }; |
Check warning
Code scanning / PMD
Unnecessary semicolon Warning test
| @@ -91,7 +92,7 @@ public void testUDFArray() { | |||
| df.printSchema(); | |||
| // register ArrayDoubleSimilarityFunction as a UDF | |||
| TestUDFDoubleArr testUDFDoubleArr = new TestUDFDoubleArr(); | |||
| SparkFnRegistrar.registerUDF2(spark, "testUDFDoubleArr", testUDFDoubleArr, DataTypes.DoubleType); | |||
| SparkFnRegistrar.registerUDF2(TestSparkBase.spark, "testUDFDoubleArr", testUDFDoubleArr, DataTypes.DoubleType); | |||
There was a problem hiding this comment.
why not inject spark instance variqable and not make changes throughout? The whole idea is the class code should not know or care about how spark session was created.
|
|
||
|
|
||
| public class TestDocumenterBase extends ZinggSparkTester { | ||
| @ExtendWith(TestSparkBase.class) | ||
| public class TestDocumenterBase { |
There was a problem hiding this comment.
we should have a ganeric test and a spark specific test for these classes
| @@ -227,7 +234,7 @@ | |||
| }); | |||
|
|
|||
|
|
|||
| Dataset<Row> sample = spark.createDataFrame(Arrays.asList( | |||
| Dataset<Row> sample = sparkSession.createDataFrame(Arrays.asList( | |||
Check warning
Code scanning / PMD
Consider simply returning the value vs storing it in local variable 'recordDF' Warning test
|
|
||
| import static org.junit.jupiter.api.Assertions.fail; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.List; | ||
|
|
||
| import org.apache.commons.io.input.TeeInputStream; |
Check warning
Code scanning / PMD
Unused import 'zingg.common.client.util.*' Warning test
| private static final int NO_OF_RECORDS = 5; | ||
| private final Dataset<Row> dataset; | ||
| private List<Row> stopwordRow; |
Check warning
Code scanning / PMD
Perhaps 'stopWords' could be replaced by a local variable. Warning test
| private static final int NO_OF_RECORDS = 5; | ||
| private final Dataset<Row> dataset; | ||
| private List<Row> stopwordRow; | ||
| private List<String> stopwordList; |
Check warning
Code scanning / PMD
Perhaps 'stopWords' could be replaced by a local variable. Warning test
| private final Dataset<Row> dataset; | ||
| private List<Row> stopwordRow; | ||
| private List<String> stopwordList; | ||
| private Dataset<Row> stopWords; |
Check warning
Code scanning / PMD
Perhaps 'stopWords' could be replaced by a local variable. Warning test
| @@ -177,7 +198,7 @@ | |||
| StructType structType = new StructType(); | |||
| structType = structType.add(DataTypes.createStructField(COL_STOPWORDS, DataTypes.StringType, false)); | |||
| //create dataframe with given records and schema | |||
| Dataset<Row> recordDF = spark.createDataFrame(records, structType); | |||
| Dataset<Row> recordDF = sparkSession.createDataFrame(records, structType); | |||
Check warning
Code scanning / PMD
Consider simply returning the value vs storing it in local variable 'recordDF' Warning test
No description provided.