Skip to content

Commit f5e4228

Browse files
committed
added spark driver memory from env variable
1 parent 2b5d6c5 commit f5e4228

File tree

1 file changed

+7
-0
lines changed

1 file changed

+7
-0
lines changed

spark/core/src/test/java/zingg/spark/core/session/SparkSessionProvider.java

+7
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,20 @@ public class SparkSessionProvider {
2323
private void initializeSession() {
2424
if (sparkSession == null) {
2525
try {
26+
String sparkDriverMemory = System.getenv("SPARK_DRIVER_MEMORY");
27+
if (sparkDriverMemory == null) {
28+
sparkDriverMemory = "1g";
29+
}
2630
sparkSession = SparkSession
2731
.builder()
2832
.master("local[*]")
2933
.appName("ZinggJunit")
3034
.config("spark.debug.maxToStringFields", 100)
35+
.config("spark.driver.memory", sparkDriverMemory)
3136
.getOrCreate();
3237
SparkContext sparkContext = sparkSession.sparkContext();
38+
long driverMemory = sparkContext.getConf().getSizeAsGb("spark.driver.memory", "0");
39+
System.out.println("Spark driver memory: " + driverMemory + " GB");
3340
if (sparkContext.getCheckpointDir().isEmpty()) {
3441
sparkContext.setCheckpointDir("/tmp/checkpoint");
3542
}

0 commit comments

Comments
 (0)