Skip to content

Commit 748fbe0

Browse files
Nitish1814Nitish1814
committed
added different output locations from single and compound tests
1 parent 70cb87e commit 748fbe0

File tree

5 files changed

+113
-7
lines changed

5 files changed

+113
-7
lines changed

‎spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsCompound.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
@ExtendWith(TestSparkBase.class)
2727
public class TestSparkExecutorsCompound extends TestExecutorsCompound<SparkSession,Dataset<Row>,Row,Column,DataType> {
28-
protected static final String CONFIG_FILE = "zingg/spark/core/executor/configSparkIntTest.json";
28+
protected static final String CONFIG_FILE = "zingg/spark/core/executor/compound/configSparkIntTest.json";
2929
protected static final String TEST_DATA_FILE = "zingg/spark/core/executor/test.csv";
3030

3131
public static final Log LOG = LogFactory.getLog(TestSparkExecutorsCompound.class);

‎spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsSingle.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929

3030
@ExtendWith(TestSparkBase.class)
3131
public class TestSparkExecutorsSingle extends TestExecutorsSingle<SparkSession,Dataset<Row>,Row,Column,DataType> {
32-
protected static final String CONFIG_FILE = "zingg/spark/core/executor/configSparkIntTest.json";
33-
protected static final String CONFIGLINK_FILE = "zingg/spark/core/executor/configSparkLinkTest.json";
32+
protected static final String CONFIG_FILE = "zingg/spark/core/executor/single/configSparkIntTest.json";
33+
protected static final String CONFIGLINK_FILE = "zingg/spark/core/executor/single/configSparkLinkTest.json";
3434
protected static final String TEST1_DATA_FILE = "zingg/spark/core/executor/test1.csv";
3535
protected static final String TEST2_DATA_FILE = "zingg/spark/core/executor/test2.csv";
3636
private final SparkSession sparkSession;

‎spark/core/src/test/resources/zingg/spark/core/executor/compound/configSparkIntTest.json

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
{
2+
"trainingSamples" : [{
3+
"name":"trainingPos",
4+
"format":"csv",
5+
"props": {
6+
"location": "./zingg/spark/core/executor/training.csv",
7+
"delimiter": ",",
8+
"header":false,
9+
"badRecordsPath":"/tmp/bad"
10+
},
11+
"schema": "z_cluster string, z_ismatch integer, id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string"
12+
}],
13+
"fieldDefinition":[
14+
{
15+
"fieldName" : "id",
16+
"matchType" : "dont_use",
17+
"fields" : "id",
18+
"dataType": "string"
19+
},
20+
{
21+
"fieldName" : "fname",
22+
"matchType" : "fuzzy",
23+
"fields" : "fname",
24+
"dataType": "string"
25+
},
26+
{
27+
"fieldName" : "lname",
28+
"matchType" : "fuzzy",
29+
"fields" : "lname",
30+
"dataType": "string"
31+
},
32+
{
33+
"fieldName" : "stNo",
34+
"matchType": "fuzzy",
35+
"fields" : "stNo",
36+
"dataType": "string"
37+
},
38+
{
39+
"fieldName" : "add1",
40+
"matchType": "fuzzy",
41+
"fields" : "add1",
42+
"dataType": "string"
43+
},
44+
{
45+
"fieldName" : "add2",
46+
"matchType": "fuzzy",
47+
"fields" : "add2",
48+
"dataType": "string"
49+
},
50+
{
51+
"fieldName" : "city",
52+
"matchType": "fuzzy",
53+
"fields" : "city",
54+
"dataType": "string"
55+
},
56+
{
57+
"fieldName" : "areacode",
58+
"matchType": "fuzzy",
59+
"fields" : "areacode",
60+
"dataType": "string"
61+
},
62+
{
63+
"fieldName" : "state",
64+
"matchType": "fuzzy",
65+
"fields" : "state",
66+
"dataType": "string"
67+
},
68+
{
69+
"fieldName" : "dob",
70+
"matchType": "fuzzy",
71+
"fields" : "dob",
72+
"dataType": "string"
73+
},
74+
{
75+
"fieldName" : "ssn",
76+
"matchType": "fuzzy",
77+
"fields" : "ssn",
78+
"dataType": "string"
79+
}
80+
],
81+
"output" : [{
82+
"name":"output",
83+
"format":"csv",
84+
"props": {
85+
"location": "/tmp/junit_integration_spark/compound/zinggOutput",
86+
"delimiter": ",",
87+
"header":true
88+
}
89+
}],
90+
"data" : [{
91+
"name":"test",
92+
"format":"csv",
93+
"props": {
94+
"location": "./zingg/spark/core/executor/test.csv",
95+
"delimiter": ",",
96+
"header":false
97+
},
98+
"schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, state string, areacode string, dob string, ssn string"
99+
}
100+
],
101+
"labelDataSampleSize" : 0.5,
102+
"numPartitions":4,
103+
"modelId": "junit_integration_spark",
104+
"zinggDir": "/tmp/junit_integration_spark/compound"
105+
106+
}

‎spark/core/src/test/resources/zingg/spark/core/executor/configSparkIntTest.json ‎spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkIntTest.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@
8282
"name":"output",
8383
"format":"csv",
8484
"props": {
85-
"location": "/tmp/junit_integration_spark/zinggOutput",
85+
"location": "/tmp/junit_integration_spark/single/zinggOutput",
8686
"delimiter": ",",
8787
"header":true
8888
}
@@ -101,6 +101,6 @@
101101
"labelDataSampleSize" : 0.5,
102102
"numPartitions":4,
103103
"modelId": "junit_integration_spark",
104-
"zinggDir": "/tmp/junit_integration_spark"
104+
"zinggDir": "/tmp/junit_integration_spark/single"
105105

106106
}

‎spark/core/src/test/resources/zingg/spark/core/executor/configSparkLinkTest.json ‎spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkLinkTest.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
"name":"output",
6666
"format":"csv",
6767
"props": {
68-
"location": "/tmp/junit_integration_spark/zinggOutput",
68+
"location": "/tmp/junit_integration_spark/single/zinggOutput",
6969
"delimiter": ",",
7070
"header":true
7171
}
@@ -94,6 +94,6 @@
9494
"labelDataSampleSize" : 0.5,
9595
"numPartitions":4,
9696
"modelId": "junit_integration_spark",
97-
"zinggDir": "/tmp/junit_integration_spark"
97+
"zinggDir": "/tmp/junit_integration_spark/single"
9898

9999
}

0 commit comments

Comments
 (0)