From bed992461210708ea4c74d320ef01b663731c29f Mon Sep 17 00:00:00 2001 From: administrator Date: Sat, 13 Jul 2024 16:02:43 +0530 Subject: [PATCH] Renamed classes and import refactors --- .../common/client/util/DFObjectUtil.java | 6 ++++ .../zingg/common/client/TestZFrameBase.java | 18 +++++------ .../data/{Constant.java => TestData.java} | 2 +- .../common/core/block/TestBlockBase.java | 3 +- .../infra/util/PojoToArrayConverter.java | 32 ------------------- .../spark/client/util/SparkDFObjectUtil.java | 5 ++- .../spark/client/util/WithSparkSession.java | 19 ----------- .../java/zingg/client/TestSparkFrame.java | 10 +++--- 8 files changed, 25 insertions(+), 70 deletions(-) rename common/client/src/test/java/zingg/common/client/data/{Constant.java => TestData.java} (99%) delete mode 100644 common/infra/src/main/java/zingg/common/infra/util/PojoToArrayConverter.java delete mode 100644 spark/client/src/main/java/zingg/spark/client/util/WithSparkSession.java diff --git a/common/client/src/main/java/zingg/common/client/util/DFObjectUtil.java b/common/client/src/main/java/zingg/common/client/util/DFObjectUtil.java index d277074b7..c0ae8bd89 100644 --- a/common/client/src/main/java/zingg/common/client/util/DFObjectUtil.java +++ b/common/client/src/main/java/zingg/common/client/util/DFObjectUtil.java @@ -6,6 +6,12 @@ public abstract class DFObjectUtil { + protected final IWithSession iWithSession; + + protected DFObjectUtil(IWithSession iWithSession) { + this.iWithSession = iWithSession; + } + public abstract ZFrame getDFFromObjectList(List objList, Class objClass) throws Exception; } diff --git a/common/client/src/test/java/zingg/common/client/TestZFrameBase.java b/common/client/src/test/java/zingg/common/client/TestZFrameBase.java index 7cdac1a9b..0735f95c3 100644 --- a/common/client/src/test/java/zingg/common/client/TestZFrameBase.java +++ b/common/client/src/test/java/zingg/common/client/TestZFrameBase.java @@ -22,15 +22,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static zingg.common.client.data.Constant.createEmptySampleData; -import static zingg.common.client.data.Constant.createSampleDataCluster; -import static zingg.common.client.data.Constant.createSampleDataClusterWithNull; -import static zingg.common.client.data.Constant.createSampleDataInput; -import static zingg.common.client.data.Constant.createSampleDataList; -import static zingg.common.client.data.Constant.createSampleDataListDistinct; -import static zingg.common.client.data.Constant.createSampleDataListWithDistinctSurnameAndPostcode; -import static zingg.common.client.data.Constant.createSampleDataListWithMixedDataType; -import static zingg.common.client.data.Constant.createSampleDataZScore; +import static zingg.common.client.data.TestData.createEmptySampleData; +import static zingg.common.client.data.TestData.createSampleDataCluster; +import static zingg.common.client.data.TestData.createSampleDataClusterWithNull; +import static zingg.common.client.data.TestData.createSampleDataInput; +import static zingg.common.client.data.TestData.createSampleDataList; +import static zingg.common.client.data.TestData.createSampleDataListDistinct; +import static zingg.common.client.data.TestData.createSampleDataListWithDistinctSurnameAndPostcode; +import static zingg.common.client.data.TestData.createSampleDataListWithMixedDataType; +import static zingg.common.client.data.TestData.createSampleDataZScore; public abstract class TestZFrameBase { diff --git a/common/client/src/test/java/zingg/common/client/data/Constant.java b/common/client/src/test/java/zingg/common/client/data/TestData.java similarity index 99% rename from common/client/src/test/java/zingg/common/client/data/Constant.java rename to common/client/src/test/java/zingg/common/client/data/TestData.java index e84f6c0fb..78a09dbb0 100644 --- a/common/client/src/test/java/zingg/common/client/data/Constant.java +++ b/common/client/src/test/java/zingg/common/client/data/TestData.java @@ -11,7 +11,7 @@ import java.util.ArrayList; import java.util.List; -public class Constant { +public class TestData { //sample data classes to be used for testing public static List createEmptySampleData() { diff --git a/common/core/src/test/java/zingg/common/core/block/TestBlockBase.java b/common/core/src/test/java/zingg/common/core/block/TestBlockBase.java index 87d340171..66b2711a0 100644 --- a/common/core/src/test/java/zingg/common/core/block/TestBlockBase.java +++ b/common/core/src/test/java/zingg/common/core/block/TestBlockBase.java @@ -52,7 +52,8 @@ public void testTree() throws Throwable { } private IArguments getArguments() throws ZinggClientException { - + String configFilePath = Objects.requireNonNull(getClass().getResource("../../../../testFebrl/config.json")).getFile(); + IArguments args = argumentsUtil.createArgumentsFromJSON(configFilePath, "trainMatch"); List fdList = getFieldDefList(); diff --git a/common/infra/src/main/java/zingg/common/infra/util/PojoToArrayConverter.java b/common/infra/src/main/java/zingg/common/infra/util/PojoToArrayConverter.java deleted file mode 100644 index a519cfe1f..000000000 --- a/common/infra/src/main/java/zingg/common/infra/util/PojoToArrayConverter.java +++ /dev/null @@ -1,32 +0,0 @@ -package zingg.common.infra.util; - -import java.lang.reflect.*; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; - -public class PojoToArrayConverter { - - public static Object[] getObjectArray(T person) throws NoSuchAlgorithmException, IllegalArgumentException, IllegalAccessException { - - List values = new ArrayList(); - if (person.getClass().getSuperclass() != null) { - Field[] fieldsSuper = person.getClass().getSuperclass().getDeclaredFields(); - if (fieldsSuper != null){ - for (Field f: fieldsSuper) { - f.setAccessible(true); - values.add(f.get(person)); - } - - } - } - Field[] fields = person.getClass().getDeclaredFields(); - - for (Field field: fields) { - field.setAccessible(true); - values.add(field.get(person)); - } - - return values.toArray(new Object[values.size()]); - } -} diff --git a/spark/client/src/main/java/zingg/spark/client/util/SparkDFObjectUtil.java b/spark/client/src/main/java/zingg/spark/client/util/SparkDFObjectUtil.java index 840dc9944..f5d185ae3 100644 --- a/spark/client/src/main/java/zingg/spark/client/util/SparkDFObjectUtil.java +++ b/spark/client/src/main/java/zingg/spark/client/util/SparkDFObjectUtil.java @@ -16,10 +16,9 @@ public class SparkDFObjectUtil extends DFObjectUtil, Row, Column> { - private final IWithSession withSparkSession; public SparkDFObjectUtil(IWithSession withSparkSession) { - this.withSparkSession = withSparkSession; + super(withSparkSession); } @Override @@ -30,7 +29,7 @@ public ZFrame, Row, Column> getDFFromObjectList(List objList, Class List rows = Arrays.asList(RowsFromObjectList.getRows(objList)); StructType structType = stpc.getStructType(objClass); - return new SparkFrame(withSparkSession.getSession().createDataFrame(rows, structType)); + return new SparkFrame(iWithSession.getSession().createDataFrame(rows, structType)); } } diff --git a/spark/client/src/main/java/zingg/spark/client/util/WithSparkSession.java b/spark/client/src/main/java/zingg/spark/client/util/WithSparkSession.java deleted file mode 100644 index 327eca0d4..000000000 --- a/spark/client/src/main/java/zingg/spark/client/util/WithSparkSession.java +++ /dev/null @@ -1,19 +0,0 @@ -package zingg.spark.client.util; - -import org.apache.spark.sql.SparkSession; -import zingg.common.client.util.WithSession; - -public class WithSparkSession implements WithSession { - - private SparkSession sparkSession; - - @Override - public void setSession(SparkSession sparkSession) { - this.sparkSession = sparkSession; - } - - @Override - public SparkSession getSession() { - return this.sparkSession; - } -} diff --git a/spark/client/src/test/java/zingg/client/TestSparkFrame.java b/spark/client/src/test/java/zingg/client/TestSparkFrame.java index f2cd6df2e..ab496a77e 100644 --- a/spark/client/src/test/java/zingg/client/TestSparkFrame.java +++ b/spark/client/src/test/java/zingg/client/TestSparkFrame.java @@ -20,10 +20,10 @@ import zingg.common.client.IArguments; import zingg.common.client.TestZFrameBase; import zingg.common.client.ZFrame; +import zingg.common.client.util.IWithSession; import zingg.common.client.util.WithSession; import zingg.spark.client.SparkFrame; import zingg.spark.client.util.SparkDFObjectUtil; -import zingg.spark.client.util.WithSparkSession; import java.util.Arrays; @@ -34,10 +34,10 @@ public class TestSparkFrame extends TestZFrameBase, R public static IArguments args; public static JavaSparkContext ctx; public static SparkSession spark; - public static WithSession withSession; + public static IWithSession iWithSession; public TestSparkFrame() { - super(new SparkDFObjectUtil(withSession)); + super(new SparkDFObjectUtil(iWithSession)); } @BeforeAll @@ -53,8 +53,8 @@ protected static void setUpSpark() { .appName("Zingg" + "Junit") .getOrCreate(); ctx = new JavaSparkContext(spark.sparkContext()); - withSession = new WithSparkSession(); - withSession.setSession(spark); + iWithSession = new WithSession<>(); + iWithSession.setSession(spark); } catch (Throwable e) { if (LOG.isDebugEnabled()) e.printStackTrace();