make Spark on Mesos use old spark shuffle fetch protocol (#85)

* make Spark on Mesos use old spark shuffle fetch protocol * bump requirements to appease github CI; bump package patch version
Yelp · Aug 25, 2022 · 4b904a9 · 4b904a9
1 parent 44cbfb2
commit 4b904a9
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 2 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
-importlib-resources==1.5.0
+importlib-resources==5.4.0
 pyinotify==0.9.6
 pyyaml >= 3.0
diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py
@@ -616,6 +616,7 @@ def _get_mesos_spark_env(
         'spark.mesos.constraints': f'pool:{paasta_pool}',
         'spark.mesos.executor.docker.forcePullImage': 'true',
         'spark.mesos.principal': 'spark',
+        'spark.shuffle.useOldFetchProtocol': 'true',
         **auth_configs,
         **_get_mesos_docker_volumes_conf(
             user_spark_opts, extra_volumes,

diff --git a/setup.py b/setup.py
@@ -17,7 +17,7 @@
 
 setup(
     name='service-configuration-lib',
-    version='2.11.0',
+    version='2.11.1',
     provides=['service_configuration_lib'],
     description='Start, stop, and inspect Yelp SOA services',
     url='https://github.com/Yelp/service_configuration_lib',

diff --git a/tests/spark_config_test.py b/tests/spark_config_test.py
@@ -929,6 +929,7 @@ def verify(output):
                 'spark.mesos.executor.docker.forcePullImage': 'true',
                 'spark.mesos.constraints': f'pool:{self.pool}',
                 'spark.mesos.principal': 'spark',
+                'spark.shuffle.useOldFetchProtocol': 'true',
             }
             for key, value in expected_output.items():
                 assert output[key] == value