Commit 7d879d8 1 parent ecfc4e7 commit 7d879d8 Copy full SHA for 7d879d8
File tree 2 files changed +16
-5
lines changed
main/java/zingg/spark/core/executor
test/java/zingg/spark/core/executor
2 files changed +16
-5
lines changed Original file line number Diff line number Diff line change 2
2
3
3
import org .apache .commons .logging .Log ;
4
4
import org .apache .commons .logging .LogFactory ;
5
+ import org .apache .spark .SparkContext ;
5
6
import org .apache .spark .api .java .JavaSparkContext ;
6
7
import org .apache .spark .sql .Column ;
7
8
import org .apache .spark .sql .Dataset ;
@@ -71,12 +72,16 @@ public void init(IZinggLicense license)
71
72
zSession = new ZSparkSession (spark , license );
72
73
}
73
74
if (ctx ==null ) {
74
- ctx = JavaSparkContext .fromSparkContext (zSession .getSession ().sparkContext ());
75
+ SparkContext sparkContext = zSession .getSession ().sparkContext ();
76
+ if (sparkContext .getCheckpointDir ().isEmpty ()) {
77
+ sparkContext .setCheckpointDir ("/tmp/checkpoint" );
78
+ }
79
+ ctx = JavaSparkContext .fromSparkContext (sparkContext );
75
80
JavaSparkContext .jarOfClass (IZingg .class );
76
81
LOG .debug ("Context " + ctx .toString ());
77
82
//initHashFns();
78
83
if (!ctx .getCheckpointDir ().isPresent ()) {
79
- ctx .setCheckpointDir ("/tmp/checkpoint" );
84
+ ctx .setCheckpointDir (sparkContext . getCheckpointDir (). get () );
80
85
}
81
86
setUtils ();
82
87
}
Original file line number Diff line number Diff line change 6
6
7
7
import org .apache .commons .logging .Log ;
8
8
import org .apache .commons .logging .LogFactory ;
9
+ import org .apache .spark .SparkContext ;
9
10
import org .apache .spark .api .java .JavaSparkContext ;
10
11
import org .apache .spark .sql .Dataset ;
11
12
import org .apache .spark .sql .Row ;
@@ -49,15 +50,20 @@ public static void setup() {
49
50
.master ("local[*]" )
50
51
.appName ("Zingg" + "Junit" )
51
52
.getOrCreate ();
52
- ctx = new JavaSparkContext (spark .sparkContext ());
53
+ SparkContext sparkContext = spark .sparkContext ();
54
+ if (sparkContext .getCheckpointDir ().isEmpty ()) {
55
+ sparkContext .setCheckpointDir ("/tmp/checkpoint" );
56
+ }
57
+ ctx = new JavaSparkContext (sparkContext );
53
58
JavaSparkContext .jarOfClass (IZingg .class );
54
59
args = new Arguments ();
55
60
zsCTX = new ZinggSparkContext ();
56
61
zsCTX .ctx = ctx ;
57
62
zSession = new ZSparkSession (spark , null );
58
63
zsCTX .zSession = zSession ;
59
-
60
- ctx .setCheckpointDir ("/tmp/checkpoint" );
64
+ if (!ctx .getCheckpointDir ().isPresent ()) {
65
+ ctx .setCheckpointDir (sparkContext .getCheckpointDir ().get ());
66
+ }
61
67
zsCTX .setPipeUtil (new SparkPipeUtil (zSession ));
62
68
zsCTX .setDSUtil (new SparkDSUtil (zSession ));
63
69
zsCTX .setHashUtil (new SparkHashUtil (zSession ));
You can’t perform that action at this time.
0 commit comments