DSP-19969 allow spark executor log to separate file

riptano · Dec 11, 2019 · c842b1f · c842b1f
1 parent f36d499
commit c842b1f
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 2 deletions.
diff --git a/bin/manager_start.sh b/bin/manager_start.sh
@@ -55,12 +55,19 @@ if [ -n "$6" ]; then
   SPARK_SUBMIT_OPTIONS="$SPARK_SUBMIT_OPTIONS --proxy-user $6"
 fi
 
+if [ "$SEPARATE_EXECUTOR_LOGGING" = "true" ]
+then
+  export EXECUTOR_LOGGING_OPTS=""
+else
+  export EXECUTOR_LOGGING_OPTS="$LOGGING_OPTS"
+fi
+
 if [ -n "$JOBSERVER_KEYTAB" ]; then
   SPARK_SUBMIT_OPTIONS="$SPARK_SUBMIT_OPTIONS --keytab $JOBSERVER_KEYTAB"
 fi
 
 cmd='$SPARK_HOME/bin/spark-submit --class $MAIN --driver-memory $JOBSERVER_MEMORY
-      --conf "spark.executor.extraJavaOptions=$LOGGING_OPTS"
+      --conf "spark.executor.extraJavaOptions=$EXECUTOR_LOGGING_OPTS"
       $SPARK_SUBMIT_OPTIONS
       --driver-java-options "$GC_OPTS $JAVA_OPTS $LOGGING_OPTS $CONFIG_OVERRIDES $SPARK_SUBMIT_JAVA_OPTIONS"
       $JAR_FILE $3 $4 $CONF_FILE'

diff --git a/bin/server_start.sh b/bin/server_start.sh
@@ -43,8 +43,15 @@ if [ -f "$PIDFILE" ] && kill -0 $(cat "$PIDFILE"); then
    exit 1
 fi
 
+if [ "$SEPARATE_EXECUTOR_LOGGING" = "true" ]
+then
+  export EXECUTOR_LOGGING_OPTS=""
+else
+  export EXECUTOR_LOGGING_OPTS="$LOGGING_OPTS"
+fi
+
 cmd='$SPARK_HOME/bin/spark-submit --class $MAIN --driver-memory $JOBSERVER_MEMORY
-  --conf "spark.executor.extraJavaOptions=$LOGGING_OPTS"
+  --conf "spark.executor.extraJavaOptions=$EXECUTOR_LOGGING_OPTS"
   --driver-java-options "$GC_OPTS $JAVA_OPTS $LOGGING_OPTS $CONFIG_OVERRIDES"
   $@ $appdir/spark-job-server.jar $conffile'