fix: handle ConnectionError in pebble-ready event (#115)

Signed-off-by: Dario Faccin <[email protected]>

src/charm.py

@@ -7,8 +7,9 @@
 import json
 import logging
 import time
+from pathlib import PurePath
 from subprocess import check_output
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from charms.kubernetes_charm_libraries.v0.hugepages_volumes_patch import (  # type: ignore[import]
     HugePagesVolume,
@@ -35,7 +36,7 @@
 from ops.charm import CharmBase, CharmEvents, CollectStatusEvent
 from ops.framework import EventBase, EventSource
 from ops.main import main
-from ops.pebble import ChangeError, ExecError, Layer, PathError
+from ops.pebble import ChangeError, ConnectionError, ExecError, Layer, PathError
 
 from charm_config import CharmConfig, CharmConfigInvalidError, CNIType, UpfMode
 from dpdk import DPDK
@@ -444,8 +445,10 @@ def _write_bessd_config_file(self, content: str) -> None:
         Args:
             content: Bessd config file content
         """
-        self._bessd_container.push(
-            path=f"{BESSD_CONTAINER_CONFIG_PATH}/{CONFIG_FILE_NAME}", source=content
+        push_file(
+            container=self._bessd_container,
+            path=f"{BESSD_CONTAINER_CONFIG_PATH}/{CONFIG_FILE_NAME}",
+            source=content,
         )
         logger.info("Pushed %s config file", CONFIG_FILE_NAME)
 
@@ -455,8 +458,9 @@ def _bessd_config_file_is_written(self) -> bool:
         Returns:
             bool: Whether the bessd config file was written
         """
-        return self._bessd_container.exists(
-            path=f"{BESSD_CONTAINER_CONFIG_PATH}/{CONFIG_FILE_NAME}"
+        return path_exists(
+            container=self._bessd_container,
+            path=f"{BESSD_CONTAINER_CONFIG_PATH}/{CONFIG_FILE_NAME}",
         )
 
     def _bessd_config_file_content_matches(self, content: str) -> bool:
@@ -465,9 +469,12 @@ def _bessd_config_file_content_matches(self, content: str) -> bool:
         Returns:
             bool: Whether the bessd config file content matches
         """
-        existing_content = self._bessd_container.pull(
-            path=f"{BESSD_CONTAINER_CONFIG_PATH}/{CONFIG_FILE_NAME}"
-        )
+        try:
+            existing_content = self._bessd_container.pull(
+                path=f"{BESSD_CONTAINER_CONFIG_PATH}/{CONFIG_FILE_NAME}"
+            )
+        except ConnectionError:
+            return False
         if existing_content.read() != content:
             return False
         return True
@@ -481,6 +488,8 @@ def _hwcksum_config_matches_pod_config(self) -> bool:
             )
         except PathError:
             existing_content = {}
+        except ConnectionError:
+            existing_content = {}
         return existing_content.get("hwcksum") != self._charm_config.enable_hw_checksum
 
     def _on_collect_unit_status(self, event: CollectStatusEvent):  # noqa C901
@@ -515,7 +524,7 @@ def _on_collect_unit_status(self, event: CollectStatusEvent):  # noqa C901
             event.add_status(WaitingStatus("Waiting for Multus to be ready"))
             logger.info("Waiting for Multus to be ready")
             return
-        if not self._bessd_container.exists(path=BESSD_CONTAINER_CONFIG_PATH):
+        if not path_exists(container=self._bessd_container, path=BESSD_CONTAINER_CONFIG_PATH):
             event.add_status(WaitingStatus("Waiting for storage to be attached"))
             logger.info("Waiting for storage to be attached")
             return
@@ -573,7 +582,7 @@ def _on_bessd_pebble_ready(self, event: EventBase) -> None:
             return
         if not self._kubernetes_multus.is_ready():
             return
-        if not self._bessd_container.exists(path=BESSD_CONTAINER_CONFIG_PATH):
+        if not path_exists(container=self._bessd_container, path=BESSD_CONTAINER_CONFIG_PATH):
             return
         self._configure_bessd_workload()
 
@@ -674,15 +683,18 @@ def _is_bessctl_executed(self) -> bool:
         Returns:
             bool:   True/False
         """
-        return self._bessd_container.exists(path=f"/{BESSCTL_CONFIGURE_EXECUTED_FILE_NAME}")
+        return path_exists(
+            container=self._bessd_container, path=f"/{BESSCTL_CONFIGURE_EXECUTED_FILE_NAME}"
+        )
 
     def _create_bessctl_executed_validation_file(self, content) -> None:
         """Create BESSCTL_CONFIGURE_EXECUTED_FILE_NAME.
 
         This must be created outside of the persistent storage volume so that
         on container restart, bessd configuration will run again.
         """
-        self._bessd_container.push(
+        push_file(
+            container=self._bessd_container,
             path=f"/{BESSCTL_CONFIGURE_EXECUTED_FILE_NAME}",
             source=content,
         )
@@ -1059,8 +1071,47 @@ def service_is_running_on_container(container: Container, service_name: str) ->
         service = container.get_service(service_name)
     except ModelError:
         return False
+    except ConnectionError:
+        return False
     return service.is_running()
 
 
+def push_file(
+    container: Container,
+    path: Union[str, PurePath],
+    source: str,
+) -> None:
+    """Pushes source content to path in container.
+
+    Args:
+        container: Container object
+        path: Path in which content is pushed
+        source: Content to be pushed to container
+    """
+    try:
+        container.push(path=path, source=source)
+    except ConnectionError:
+        return
+
+
+def path_exists(
+    container: Container,
+    path: Union[str, PurePath],
+) -> bool:
+    """Returns existence of path in container.
+
+    Args:
+        container: Container object
+        path: Path to verify the existence of
+
+    Returns:
+        bool: existence of path in container
+    """
+    try:
+        return container.exists(path=path)
+    except ConnectionError:
+        return False
+
+
 if __name__ == "__main__":  # pragma: no cover
     main(UPFOperatorCharm)

tests/unit/test_charm.py

@@ -14,6 +14,7 @@
 from lightkube.models.meta_v1 import ObjectMeta
 from lightkube.resources.core_v1 import Service
 from ops import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus, testing
+from ops.pebble import ConnectionError
 
 from charm import (
     ACCESS_INTERFACE_NAME,
@@ -742,6 +743,23 @@ def test_given_cant_connect_to_bessd_container_when_pfcp_agent_pebble_ready_then
             WaitingStatus("Waiting for bessd container to be ready"),
         )
 
+    @patch("ops.model.Container.get_service")
+    @patch(f"{MULTUS_LIBRARY_PATH}.KubernetesMultusCharmLib.is_ready")
+    def test_given_pebble_connection_error_when_bessd_pebble_ready_then_status_is_waiting(  # noqa: E501
+        self, patch_is_ready, patch_get_service
+    ):
+        self.harness.handle_exec("bessd", [], result=0)
+        patch_get_service.side_effect = ConnectionError()
+        patch_is_ready.return_value = True
+        self.harness.set_can_connect(container="bessd", val=True)
+
+        self.harness.container_pebble_ready(container_name="bessd")
+        self.harness.evaluate_status()
+
+        self.assertEqual(
+            self.harness.model.unit.status, WaitingStatus("Waiting for bessd service to run")
+        )
+
     @patch(f"{HUGEPAGES_LIBRARY_PATH}.KubernetesHugePagesPatchCharmLib.is_patched")
     @patch("charms.sdcore_upf_k8s.v0.fiveg_n3.N3Provides.publish_upf_information")
     def test_given_fiveg_n3_relation_created_when_fiveg_n3_request_then_upf_ip_address_is_published(  # noqa: E501