diff --git a/paasta_tools/cli/cmds/spark_run.py b/paasta_tools/cli/cmds/spark_run.py index 8356a4c0b9..568cbd45ba 100644 --- a/paasta_tools/cli/cmds/spark_run.py +++ b/paasta_tools/cli/cmds/spark_run.py @@ -313,9 +313,8 @@ def add_subparser(subparsers): list_parser.add_argument( "--enable-dra", help=( - "Enable Dynamic Resource Allocation (DRA) for the Spark job as documented in (y/spark-dra). DRA " - "dynamically scales up and down the executor instance count based on the number of pending tasks " - "and requirements. Disabled by default. Does not override Spark DRA configs if specified by the user." + "[DEPRECATED] Enable Dynamic Resource Allocation (DRA) for the Spark job as documented in (y/spark-dra)." + "DRA is enabled by default now. This config is a no-op operation and recommended to be removed." ), action="store_true", default=False, @@ -1255,13 +1254,6 @@ def paasta_spark_run(args): "spark.kubernetes.decommission.script" ] = "/opt/spark/kubernetes/dockerfiles/spark/decom.sh" - # Experimental: TODO: Move to service_configuration_lib once confirmed that there are no issues - # Enable AQE: Adaptive Query Execution - if "spark.sql.adaptive.enabled" not in spark_conf: - spark_conf["spark.sql.adaptive.enabled"] = "true" - aqe_msg = "Spark performance improving feature Adaptive Query Execution (AQE) is enabled. Set spark.sql.adaptive.enabled as false to disable." - log.info(aqe_msg) - print(PaastaColors.blue(aqe_msg)) return configure_and_run_docker_container( args, docker_img=docker_image_digest, diff --git a/tests/cli/test_cmds_spark_run.py b/tests/cli/test_cmds_spark_run.py index ab4c7161f8..adbceeb01c 100644 --- a/tests/cli/test_cmds_spark_run.py +++ b/tests/cli/test_cmds_spark_run.py @@ -1180,7 +1180,6 @@ def test_paasta_spark_run_bash( k8s_server_address=None, ) mock_spark_conf = mock_get_spark_conf.return_value - mock_spark_conf["spark.sql.adaptive.enabled"] = "true" mock_configure_and_run_docker_container.assert_called_once_with( args, docker_img=DUMMY_DOCKER_IMAGE_DIGEST,