Actually fix pod name length

src/zenml/integrations/kubernetes/orchestrators/kube_utils.py

@@ -94,11 +94,12 @@ def load_kube_config(
         k8s_config.load_kube_config(context=context)
 
 
-def sanitize_pod_name(pod_name: str) -> str:
+def sanitize_pod_name(pod_name: str, namespace: str) -> str:
     """Sanitize pod names so they conform to Kubernetes pod naming convention.
 
     Args:
         pod_name: Arbitrary input pod name.
+        namespace: Namespace in which the Pod will be created.
 
     Returns:
         Sanitized pod name.
@@ -109,7 +110,14 @@ def sanitize_pod_name(pod_name: str) -> str:
     pod_name = re.sub(r"[-]+$", "", pod_name)
     pod_name = re.sub(r"[-]+", "-", pod_name)
 
-    return pod_name[:253]
+    # Kubernetes allows Pod names to have 253 characters. However, when
+    # creating a pod they try to create a log file which is called
+    # <NAMESPACE>_<POD_NAME>_<UUID>, which adds additional characters and
+    # runs into filesystem limitations for filename lengths. We therefore
+    # subtract the length of a UUID (36), the two underscores and the
+    # namespace length from the pod name.
+    allowed_length = 253 - 38 - len(namespace)
+    return pod_name[:allowed_length]
 
 
 def sanitize_label(label: str) -> str:

src/zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py

@@ -395,10 +395,16 @@ def prepare_or_run_pipeline(
                 )
 
         pipeline_name = deployment.pipeline_configuration.name
+
+        # See `kube_utils.sanitize_pod_name` for an explanation of this crazy
+        # calculation
+        max_run_name_length = 253 - 38 - len(self.config.kubernetes_namespace)
         orchestrator_run_name = get_orchestrator_run_name(
-            pipeline_name, max_length=253
+            pipeline_name, max_length=max_run_name_length
+        )
+        pod_name = kube_utils.sanitize_pod_name(
+            orchestrator_run_name, namespace=self.config.kubernetes_namespace
         )
-        pod_name = kube_utils.sanitize_pod_name(orchestrator_run_name)
 
         assert stack.container_registry
 

src/zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py

@@ -197,7 +197,9 @@ def launch(
         )
 
         pod_name = f"{info.run_name}_{info.pipeline_step_name}"
-        pod_name = kube_utils.sanitize_pod_name(pod_name)
+        pod_name = kube_utils.sanitize_pod_name(
+            pod_name, namespace=self.config.kubernetes_namespace
+        )
 
         command = entrypoint_command[:3]
         args = entrypoint_command[3:]

src/zenml/pipelines/build_utils.py

@@ -353,7 +353,7 @@ def create_pipeline_build(
             dockerfile = images[item_key].dockerfile
             requirements = images[item_key].requirements
         else:
-            tag = deployment.pipeline_configuration.name
+            tag = deployment.pipeline_configuration.name[:16]
             if build_config.step_name:
                 tag += f"-{build_config.step_name}"
             tag += f"-{build_config.key}"