From c249b3befc3ddd9885d48ec44493d1de8e92d188 Mon Sep 17 00:00:00 2001 From: Manpreet Singh Date: Fri, 16 Jun 2023 02:19:48 -0700 Subject: [PATCH 1/2] Fix DRA comment and clean-up adaptive query execution config --- paasta_tools/cli/cmds/spark_run.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/paasta_tools/cli/cmds/spark_run.py b/paasta_tools/cli/cmds/spark_run.py index 78ed587d28..e43a8c1ce7 100644 --- a/paasta_tools/cli/cmds/spark_run.py +++ b/paasta_tools/cli/cmds/spark_run.py @@ -313,9 +313,8 @@ def add_subparser(subparsers): list_parser.add_argument( "--enable-dra", help=( - "Enable Dynamic Resource Allocation (DRA) for the Spark job as documented in (y/spark-dra). DRA " - "dynamically scales up and down the executor instance count based on the number of pending tasks " - "and requirements. Disabled by default. Does not override Spark DRA configs if specified by the user." + "[DEPRECATED] Enable Dynamic Resource Allocation (DRA) for the Spark job as documented in (y/spark-dra)." + "DRA is enabled by default now. This config is a no-op operation and recommended to be removed." ), action="store_true", default=False, @@ -1221,13 +1220,6 @@ def paasta_spark_run(args): "spark.kubernetes.decommission.script" ] = "/opt/spark/kubernetes/dockerfiles/spark/decom.sh" - # Experimental: TODO: Move to service_configuration_lib once confirmed that there are no issues - # Enable AQE: Adaptive Query Execution - if "spark.sql.adaptive.enabled" not in spark_conf: - spark_conf["spark.sql.adaptive.enabled"] = "true" - aqe_msg = "Spark performance improving feature Adaptive Query Execution (AQE) is enabled. Set spark.sql.adaptive.enabled as false to disable." - log.info(aqe_msg) - print(PaastaColors.blue(aqe_msg)) return configure_and_run_docker_container( args, docker_img=docker_image_digest, From cfa7e30578e587fe1c30d94812f6903d606a53f0 Mon Sep 17 00:00:00 2001 From: Manpreet Singh Date: Fri, 16 Jun 2023 02:20:21 -0700 Subject: [PATCH 2/2] Fix DRA comment and clean-up adaptive query execution config --- tests/cli/test_cmds_spark_run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cli/test_cmds_spark_run.py b/tests/cli/test_cmds_spark_run.py index af3b42961d..b73d0d3cf0 100644 --- a/tests/cli/test_cmds_spark_run.py +++ b/tests/cli/test_cmds_spark_run.py @@ -1179,7 +1179,6 @@ def test_paasta_spark_run_bash( k8s_server_address=None, ) mock_spark_conf = mock_get_spark_conf.return_value - mock_spark_conf["spark.sql.adaptive.enabled"] = "true" mock_configure_and_run_docker_container.assert_called_once_with( args, docker_img=DUMMY_DOCKER_IMAGE_DIGEST,