diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 0d393ae..301e150 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -265,6 +265,7 @@ def _get_k8s_spark_env( docker_img: str, volumes: Optional[List[Mapping[str, str]]], paasta_pool: str, + driver_ui_port: int, service_account_name: Optional[str] = None, include_self_managed_configs: bool = True, k8s_server_address: Optional[str] = None, @@ -294,6 +295,7 @@ def _get_k8s_spark_env( 'spark.kubernetes.executor.label.paasta.yelp.com/instance': _paasta_instance, 'spark.kubernetes.executor.label.paasta.yelp.com/cluster': _paasta_cluster, 'spark.kubernetes.executor.label.spark.yelp.com/user': user, + 'spark.kubernetes.executor.label.spark.yelp.com/driver_ui_port': str(driver_ui_port), 'spark.kubernetes.node.selector.yelp.com/pool': paasta_pool, 'spark.kubernetes.executor.label.yelp.com/pool': paasta_pool, 'spark.kubernetes.executor.label.paasta.yelp.com/pool': paasta_pool, @@ -1062,8 +1064,9 @@ def get_spark_conf( spark_app_base_name ) - ui_port = (spark_opts_from_env or {}).get('spark.ui.port') or _pick_random_port( - PREFERRED_SPARK_UI_PORT, + ui_port = int( + (spark_opts_from_env or {}).get('spark.ui.port') or + _pick_random_port(PREFERRED_SPARK_UI_PORT), ) # app_name from env is already appended port and time to make it unique @@ -1096,6 +1099,7 @@ def get_spark_conf( docker_img, extra_volumes, paasta_pool, + ui_port, service_account_name=service_account_name, include_self_managed_configs=not use_eks, k8s_server_address=k8s_server_address, diff --git a/setup.py b/setup.py index d1d205f..1de2655 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='service-configuration-lib', - version='2.18.2', + version='2.18.3', provides=['service_configuration_lib'], description='Start, stop, and inspect Yelp SOA services', url='https://github.com/Yelp/service_configuration_lib', diff --git a/tests/spark_config_test.py b/tests/spark_config_test.py index d5278b6..92b46aa 100644 --- a/tests/spark_config_test.py +++ b/tests/spark_config_test.py @@ -1168,7 +1168,9 @@ def _get_k8s_base_volumes(self): ] @pytest.fixture - def assert_kubernetes_conf(self, base_volumes): + def assert_kubernetes_conf(self, base_volumes, ui_port, mock_pick_random_port): + expected_ui_port = ui_port if ui_port else mock_pick_random_port + expected_output = { 'spark.master': f'k8s://https://k8s.{self.cluster}.paasta:6443', 'spark.executorEnv.PAASTA_SERVICE': self.service, @@ -1191,6 +1193,7 @@ def assert_kubernetes_conf(self, base_volumes): 'spark.kubernetes.executor.label.paasta.yelp.com/instance': self.instance, 'spark.kubernetes.executor.label.paasta.yelp.com/cluster': self.cluster, 'spark.kubernetes.executor.label.spark.yelp.com/user': TEST_USER, + 'spark.kubernetes.executor.label.spark.yelp.com/driver_ui_port': str(expected_ui_port), 'spark.kubernetes.node.selector.yelp.com/pool': self.pool, 'spark.kubernetes.executor.label.yelp.com/pool': self.pool, 'spark.kubernetes.executor.label.paasta.yelp.com/pool': self.pool,