[k8s jobsets] add startup policy. (Netflix#2063)

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -543,7 +543,11 @@ def _save_package_once(cls, flow_datastore, package):
 
 # TODO: Unify this method with the multi-node setup in @batch
 def _setup_multinode_environment():
-    # FIXME: what about MF_MASTER_PORT
+    # TODO [FIXME SOON]
+    # Even if Kubernetes may deploy control pods before worker pods, there is always a
+    # possibility that the worker pods may start before the control. In the case that this happens,
+    # the worker pods will not be able to resolve the control pod's IP address and this will cause
+    # the worker pods to fail. This function should account for this in the near future.
     import socket
 
     try:

metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -866,7 +866,13 @@ def dump(self):
             spec=dict(
                 replicatedJobs=[self.control.dump(), self.worker.dump()],
                 suspend=False,
-                startupPolicy=None,
+                startupPolicy=dict(
+                    # We explicitly set an InOrder Startup policy so that
+                    # we can ensure that the control pod starts before the worker pods.
+                    # This is required so that when worker pods try to access the control's IP
+                    # we are able to resolve the control's IP address.
+                    startupPolicyOrder="InOrder"
+                ),
                 successPolicy=None,
                 # The Failure Policy helps setting the number of retries for the jobset.
                 # but we don't rely on it and instead rely on either the local scheduler