diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 7568075..7a15b71 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -1109,11 +1109,13 @@ def get_spark_conf( spark_app_base_name ) - # Pick a port from a pre-defined port range, which will then be used by our Jupyter - # server metric aggregator API. The aggregator API collects Prometheus metrics from multiple - # Spark sessions and exposes them through a single endpoint. - ui_port: int = self.spark_constants.get('preferred_spark_ui_port_start', EPHEMERAL_PORT_START) - if not self.is_driver_on_k8s_tron: + if self.is_driver_on_k8s_tron: + # For Tron-launched driver on k8s, we use a static port (EPHEMERAL_PORT_START) for Spark UI + ui_port: int = self.spark_constants.get('preferred_spark_ui_port_start', EPHEMERAL_PORT_START) + else: + # Pick a port from a pre-defined port range, which will then be used by our Jupyter + # server metric aggregator API. The aggregator API collects Prometheus metrics from multiple + # Spark sessions and exposes them through a single endpoint. try: ui_port = int( (spark_opts_from_env or {}).get('spark.ui.port') or