Add driver ui port executor pod label for prometheus metrics (#123)

service_configuration_lib/spark_config.py

@@ -265,6 +265,7 @@ def _get_k8s_spark_env(
     docker_img: str,
     volumes: Optional[List[Mapping[str, str]]],
     paasta_pool: str,
+    driver_ui_port: int,
     service_account_name: Optional[str] = None,
     include_self_managed_configs: bool = True,
     k8s_server_address: Optional[str] = None,
@@ -294,6 +295,7 @@ def _get_k8s_spark_env(
         'spark.kubernetes.executor.label.paasta.yelp.com/instance': _paasta_instance,
         'spark.kubernetes.executor.label.paasta.yelp.com/cluster': _paasta_cluster,
         'spark.kubernetes.executor.label.spark.yelp.com/user': user,
+        'spark.kubernetes.executor.label.spark.yelp.com/driver_ui_port': str(driver_ui_port),
         'spark.kubernetes.node.selector.yelp.com/pool': paasta_pool,
         'spark.kubernetes.executor.label.yelp.com/pool': paasta_pool,
         'spark.kubernetes.executor.label.paasta.yelp.com/pool': paasta_pool,
@@ -1062,8 +1064,9 @@ def get_spark_conf(
             spark_app_base_name
         )
 
-        ui_port = (spark_opts_from_env or {}).get('spark.ui.port') or _pick_random_port(
-            PREFERRED_SPARK_UI_PORT,
+        ui_port = int(
+            (spark_opts_from_env or {}).get('spark.ui.port') or
+            _pick_random_port(PREFERRED_SPARK_UI_PORT),
         )
 
         # app_name from env is already appended port and time to make it unique
@@ -1096,6 +1099,7 @@ def get_spark_conf(
                 docker_img,
                 extra_volumes,
                 paasta_pool,
+                ui_port,
                 service_account_name=service_account_name,
                 include_self_managed_configs=not use_eks,
                 k8s_server_address=k8s_server_address,

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='service-configuration-lib',
-    version='2.18.2',
+    version='2.18.3',
     provides=['service_configuration_lib'],
     description='Start, stop, and inspect Yelp SOA services',
     url='https://github.com/Yelp/service_configuration_lib',

tests/spark_config_test.py

@@ -1168,7 +1168,9 @@ def _get_k8s_base_volumes(self):
         ]
 
     @pytest.fixture
-    def assert_kubernetes_conf(self, base_volumes):
+    def assert_kubernetes_conf(self, base_volumes, ui_port, mock_pick_random_port):
+        expected_ui_port = ui_port if ui_port else mock_pick_random_port
+
         expected_output = {
             'spark.master': f'k8s://https://k8s.{self.cluster}.paasta:6443',
             'spark.executorEnv.PAASTA_SERVICE': self.service,
@@ -1191,6 +1193,7 @@ def assert_kubernetes_conf(self, base_volumes):
             'spark.kubernetes.executor.label.paasta.yelp.com/instance': self.instance,
             'spark.kubernetes.executor.label.paasta.yelp.com/cluster': self.cluster,
             'spark.kubernetes.executor.label.spark.yelp.com/user': TEST_USER,
+            'spark.kubernetes.executor.label.spark.yelp.com/driver_ui_port': str(expected_ui_port),
             'spark.kubernetes.node.selector.yelp.com/pool': self.pool,
             'spark.kubernetes.executor.label.yelp.com/pool': self.pool,
             'spark.kubernetes.executor.label.paasta.yelp.com/pool': self.pool,