TaskSetManager is logs correctly, for executors so we can disable Log…

…ErrorSparkListener. #897
Open-EO · Oct 8, 2024 · 79811d5 · 79811d5
1 parent adecba9
commit 79811d5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/openeogeotrellis/deploy/sparkapplication_template.yaml.j2 b/openeogeotrellis/deploy/sparkapplication_template.yaml.j2
@@ -58,7 +58,7 @@ spec:
     "spark.kryoserializer.buffer.max": "1g"
     "spark.kryo.classesToRegister": "org.openeo.geotrellis.layers.BandCompositeRasterSource,geotrellis.raster.RasterRegion,geotrellis.raster.geotiff.GeoTiffResampleRasterSource,geotrellis.raster.RasterSource,geotrellis.raster.SourceName,geotrellis.raster.geotiff.GeoTiffPath"
     "spark.serializer": "org.apache.spark.serializer.KryoSerializer"
-    "spark.extraListeners": "org.openeo.sparklisteners.LogErrorSparkListener,org.openeo.sparklisteners.BatchJobProgressListener"
+    "spark.extraListeners": "org.openeo.sparklisteners.BatchJobProgressListener"
     "spark.dynamicAllocation.enabled": "true"
     "spark.dynamicAllocation.shuffleTracking.enabled": "true"
     "spark.dynamicAllocation.executorIdleTimeout": "240s"

diff --git a/openeogeotrellis/deploy/submit_batch_job_spark3.sh b/openeogeotrellis/deploy/submit_batch_job_spark3.sh
@@ -191,7 +191,7 @@ spark-submit \
  --conf spark.history.store.path=/var/lib/spark2/shs_db \
  --conf spark.yarn.historyServer.address=epod-ha.vgt.vito.be:18481 \
  --conf spark.archives=${archives} \
- --conf spark.extraListeners=org.openeo.sparklisteners.LogErrorSparkListener,org.openeo.sparklisteners.BatchJobProgressListener \
+ --conf spark.extraListeners=org.openeo.sparklisteners.BatchJobProgressListener \
  --conf spark.sql.adaptive.coalescePartitions.parallelismFirst=false \
  --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=5242880 \
  --files "${files}" \