From 79811d5ebf24ff8bd53840e1508e7885a1922bca Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 8 Oct 2024 18:05:58 +0200 Subject: [PATCH] =?UTF-8?q?TaskSetManager=20is=20logs=20correctly,=20for?= =?UTF-8?q?=20executors=20so=20we=20can=20disable=20Log=E2=80=A6=20?= =?UTF-8?q?=E2=80=A6ErrorSparkListener.=20https://github.com/Open-EO/opene?= =?UTF-8?q?o-geopyspark-driver/issues/897?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openeogeotrellis/deploy/sparkapplication_template.yaml.j2 | 2 +- openeogeotrellis/deploy/submit_batch_job_spark3.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openeogeotrellis/deploy/sparkapplication_template.yaml.j2 b/openeogeotrellis/deploy/sparkapplication_template.yaml.j2 index 4a26a8cc..c3d8a933 100644 --- a/openeogeotrellis/deploy/sparkapplication_template.yaml.j2 +++ b/openeogeotrellis/deploy/sparkapplication_template.yaml.j2 @@ -58,7 +58,7 @@ spec: "spark.kryoserializer.buffer.max": "1g" "spark.kryo.classesToRegister": "org.openeo.geotrellis.layers.BandCompositeRasterSource,geotrellis.raster.RasterRegion,geotrellis.raster.geotiff.GeoTiffResampleRasterSource,geotrellis.raster.RasterSource,geotrellis.raster.SourceName,geotrellis.raster.geotiff.GeoTiffPath" "spark.serializer": "org.apache.spark.serializer.KryoSerializer" - "spark.extraListeners": "org.openeo.sparklisteners.LogErrorSparkListener,org.openeo.sparklisteners.BatchJobProgressListener" + "spark.extraListeners": "org.openeo.sparklisteners.BatchJobProgressListener" "spark.dynamicAllocation.enabled": "true" "spark.dynamicAllocation.shuffleTracking.enabled": "true" "spark.dynamicAllocation.executorIdleTimeout": "240s" diff --git a/openeogeotrellis/deploy/submit_batch_job_spark3.sh b/openeogeotrellis/deploy/submit_batch_job_spark3.sh index 3d965d20..7f6e1f1a 100755 --- a/openeogeotrellis/deploy/submit_batch_job_spark3.sh +++ b/openeogeotrellis/deploy/submit_batch_job_spark3.sh @@ -191,7 +191,7 @@ spark-submit \ --conf spark.history.store.path=/var/lib/spark2/shs_db \ --conf spark.yarn.historyServer.address=epod-ha.vgt.vito.be:18481 \ --conf spark.archives=${archives} \ - --conf spark.extraListeners=org.openeo.sparklisteners.LogErrorSparkListener,org.openeo.sparklisteners.BatchJobProgressListener \ + --conf spark.extraListeners=org.openeo.sparklisteners.BatchJobProgressListener \ --conf spark.sql.adaptive.coalescePartitions.parallelismFirst=false \ --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=5242880 \ --files "${files}" \