Using ramdisk as LXD storage for the runners

config.yaml

@@ -33,13 +33,14 @@ options:
     type: string
     default: 7GiB
     description: >
-      Amount of memory to allocate per virtual machine runner. Positive integers with GiB suffix.
+      Amount of memory to allocate per virtual machine runner. Positive integers with KiB, MiB, GiB,
+      TiB, PiB, EiB suffix.
   vm-disk:
     type: string
     default: 10GiB
     description: >
       Amount of disk space to allocate to root disk for virtual machine runner. Positive integers
-      with GiB suffix.
+      with KiB, MiB, GiB, TiB, PiB, EiB suffix.
   reconcile-interval:
     type: int
     default: 10

metadata.yaml

@@ -10,14 +10,13 @@ issues: https://github.com/canonical/github-runner-operator/issues
 source: https://github.com/canonical/github-runner-operator
 summary: Creates a cluster of self-hosted github runners.
 description: |
-  A [Juju](https://juju.is/) [charm](https://juju.is/docs/olm/charmed-operators)
-  deploying self-hosted GitHub runners.
-
-  Each unit of this charm will start a configurable number of LXD based containers
-  and virtual machines to host them. Each runner performs only one job, after which
-  it unregisters from GitHub to ensure that each job runs in a clean environment.
-  The charm will periodically check the number of idle runners and spawn or destroy them as
-  necessary to maintain the configured number of runners. Both the reconciliation interval and the
-  number of runners to maintain are configurable.
+  A [Juju](https://juju.is/) [charm](https://juju.is/docs/olm/charmed-operators) deploying
+  self-hosted GitHub runners.
+
+  Each unit of this charm will start a configurable number of LXD based virtual machines to host
+  them. Each runner performs only one job, after which it unregisters from GitHub to ensure that
+  each job runs in a clean environment. The charm will periodically check the number of idle runners
+  and spawn or destroy them as necessary to maintain the configured number of runners. Both the
+  reconciliation interval and the number of runners to maintain are configurable.
 series:
-    - jammy
+  - jammy

script/deploy_runner.sh

@@ -30,5 +30,5 @@ unzip -p github_runner.zip > github-runner.charm
 rm github_runner.zip
 
 # Deploy the charm.
-juju deploy ./github-runner.charm --series=jammy e2e-runner
+juju deploy ./github-runner.charm --series=jammy --constraints="cores=4 mem=32G" e2e-runner
 juju config e2e-runner token="$GITHUB_TOKEN" path=canonical/github-runner-operator virtual-machines=1

src/charm.py

@@ -27,12 +27,12 @@
 from ops.main import main
 from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus
 
-from errors import RunnerError, SubprocessError
+from errors import MissingConfigurationError, RunnerError, SubprocessError
 from event_timer import EventTimer, TimerDisableError, TimerEnableError
 from github_type import GitHubRunnerStatus
 from runner_manager import RunnerManager, RunnerManagerConfig
 from runner_type import GitHubOrg, GitHubRepo, ProxySetting, VirtualMachineResources
-from utilities import execute_command, get_env_var, retry
+from utilities import execute_command, get_env_var, retry, secure_run_subprocess
 
 if TYPE_CHECKING:
     from ops.model import JsonObject  # pragma: no cover
@@ -52,9 +52,7 @@ class UpdateRunnerBinEvent(EventBase):
 EventT = TypeVar("EventT")
 
 
-def catch_unexpected_charm_errors(
-    func: Callable[[CharmT, EventT], None]
-) -> Callable[[CharmT, EventT], None]:
+def catch_charm_errors(func: Callable[[CharmT, EventT], None]) -> Callable[[CharmT, EventT], None]:
     """Catch unexpected errors in charm.
 
     This decorator is for unrecoverable errors and sets the charm to
@@ -72,14 +70,16 @@ def func_with_catch_unexpected_errors(self, event: EventT) -> None:
         # Safe guard against unexpected error.
         try:
             func(self, event)
-        except Exception as err:  # pylint: disable=broad-exception-caught
-            logger.exception(err)
-            self.unit.status = BlockedStatus(str(err))
+        except MissingConfigurationError as err:
+            logger.exception("Missing required charm configuration")
+            self.unit.status = BlockedStatus(
+                f"Missing required charm configuration: {err.configs}"
+            )
 
     return func_with_catch_unexpected_errors
 
 
-def catch_unexpected_action_errors(
+def catch_action_errors(
     func: Callable[[CharmT, ActionEvent], None]
 ) -> Callable[[CharmT, ActionEvent], None]:
     """Catch unexpected errors in actions.
@@ -92,15 +92,15 @@ def catch_unexpected_action_errors(
     """
 
     @functools.wraps(func)
-    def func_with_catch_unexpected_errors(self, event: ActionEvent) -> None:
+    def func_with_catch_errors(self, event: ActionEvent) -> None:
         # Safe guard against unexpected error.
         try:
             func(self, event)
-        except Exception as err:  # pylint: disable=broad-exception-caught
-            logger.exception(err)
-            event.fail(f"Failed to get runner info: {err}")
+        except MissingConfigurationError as err:
+            logger.exception("Missing required charm configuration")
+            event.fail(f"Missing required charm configuration: {err.configs}")
 
-    return func_with_catch_unexpected_errors
+    return func_with_catch_errors
 
 
 class GithubRunnerCharm(CharmBase):
@@ -112,6 +112,7 @@ class GithubRunnerCharm(CharmBase):
     repo_check_web_service_path = Path("/home/ubuntu/repo_policy_compliance_service")
     repo_check_web_service_script = Path("src/repo_policy_compliance_service.py")
     repo_check_systemd_service = Path("/etc/systemd/system/repo-policy-compliance.service")
+    lvm_vg_name = "ramdisk_pool"
 
     def __init__(self, *args, **kargs) -> None:
         """Construct the charm.
@@ -155,9 +156,24 @@ def __init__(self, *args, **kargs) -> None:
         self.framework.observe(self.on.flush_runners_action, self._on_flush_runners_action)
         self.framework.observe(self.on.update_runner_bin_action, self._on_update_runner_bin)
 
+    def _ensure_ramdisk_volume_group(self) -> None:
+        """Ensure ramdisk LVM volume group exists."""
+        # Check if ramdisk at /dev/ram0 exists.
+        result = secure_run_subprocess(["test", "-e", "/dev/ram0"])
+        if result.returncode != 0:
+            # The block ram disk is set to 1 TiB size, as a way to not limit it.
+            # Block ram disk does not pre-allocate the memory.
+            # Each LXD instance memory usage is restricted through the LXD profile.
+            execute_command(["modprobe", "brd", "rd_size=1073741824", "rd_nr=1"])
+
+        # Check if volume group exits.
+        result = secure_run_subprocess(["vgdisplay", self.lvm_vg_name])
+        if result.returncode != 0:
+            execute_command(["vgcreate", self.lvm_vg_name, "/dev/ram0"])
+
     def _get_runner_manager(
         self, token: Optional[str] = None, path: Optional[str] = None
-    ) -> Optional[RunnerManager]:
+    ) -> RunnerManager:
         """Get a RunnerManager instance, or None if missing config.
 
         Args:
@@ -166,15 +182,22 @@ def _get_runner_manager(
                 name.
 
         Returns:
-            A instance of RunnerManager if the token and path configuration can be found.
+            A instance of RunnerManager.
         """
         if token is None:
             token = self.config["token"]
         if path is None:
             path = self.config["path"]
 
-        if not token or not path:
-            return None
+        missing_configs = []
+        if not token:
+            missing_configs.append("token")
+        if not path:
+            missing_configs.append("path")
+        if missing_configs:
+            raise MissingConfigurationError(missing_configs)
+
+        self._ensure_ramdisk_volume_group()
 
         if self.service_token is None:
             self.service_token = self._get_service_token()
@@ -194,11 +217,11 @@ def _get_runner_manager(
         return RunnerManager(
             app_name,
             unit,
-            RunnerManagerConfig(path, token, "jammy", self.service_token),
+            RunnerManagerConfig(path, token, "jammy", self.service_token, self.lvm_vg_name),
             proxies=self.proxies,
         )
 
-    @catch_unexpected_charm_errors
+    @catch_charm_errors
     def _on_install(self, _event: InstallEvent) -> None:
         """Handle the installation of charm.
 
@@ -244,7 +267,7 @@ def _on_install(self, _event: InstallEvent) -> None:
         else:
             self.unit.status = BlockedStatus("Missing token or org/repo path config")
 
-    @catch_unexpected_charm_errors
+    @catch_charm_errors
     def _on_upgrade_charm(self, _event: UpgradeCharmEvent) -> None:
         """Handle the update of charm.
 
@@ -263,7 +286,7 @@ def _on_upgrade_charm(self, _event: UpgradeCharmEvent) -> None:
         runner_manager.flush()
         self._reconcile_runners(runner_manager)
 
-    @catch_unexpected_charm_errors
+    @catch_charm_errors
     def _on_config_changed(self, _event: ConfigChangedEvent) -> None:
         """Handle the configuration change.
 
@@ -298,7 +321,7 @@ def _on_config_changed(self, _event: ConfigChangedEvent) -> None:
         else:
             self.unit.status = BlockedStatus("Missing token or org/repo path config")
 
-    @catch_unexpected_charm_errors
+    @catch_charm_errors
     def _on_update_runner_bin(self, _event: UpdateRunnerBinEvent) -> None:
         """Handle checking update of runner binary event.
 
@@ -337,7 +360,7 @@ def _on_update_runner_bin(self, _event: UpdateRunnerBinEvent) -> None:
 
         self.unit.status = ActiveStatus()
 
-    @catch_unexpected_charm_errors
+    @catch_charm_errors
     def _on_reconcile_runners(self, _event: ReconcileRunnersEvent) -> None:
         """Handle the reconciliation of runners.
 
@@ -363,7 +386,7 @@ def _on_reconcile_runners(self, _event: ReconcileRunnersEvent) -> None:
 
         self.unit.status = ActiveStatus()
 
-    @catch_unexpected_action_errors
+    @catch_action_errors
     def _on_check_runners_action(self, event: ActionEvent) -> None:
         """Handle the action of checking of runner state.
 
@@ -403,7 +426,7 @@ def _on_check_runners_action(self, event: ActionEvent) -> None:
             }
         )
 
-    @catch_unexpected_action_errors
+    @catch_action_errors
     def _on_reconcile_runners_action(self, event: ActionEvent) -> None:
         """Handle the action of reconcile of runner state.
 
@@ -420,7 +443,7 @@ def _on_reconcile_runners_action(self, event: ActionEvent) -> None:
         self._on_check_runners_action(event)
         event.set_results(delta)
 
-    @catch_unexpected_action_errors
+    @catch_action_errors
     def _on_flush_runners_action(self, event: ActionEvent) -> None:
         """Handle the action of flushing all runner and reconciling afterwards.
 
@@ -438,7 +461,7 @@ def _on_flush_runners_action(self, event: ActionEvent) -> None:
         self._on_check_runners_action(event)
         event.set_results(delta)
 
-    @catch_unexpected_charm_errors
+    @catch_charm_errors
     def _on_stop(self, _: StopEvent) -> None:
         """Handle the stopping of the charm.
 
@@ -483,7 +506,7 @@ def _reconcile_runners(self, runner_manager: RunnerManager) -> Dict[str, "JsonOb
             return {"delta": {"virtual-machines": delta_virtual_machines}}
         # Safe guard against transient unexpected error.
         except Exception as err:  # pylint: disable=broad-exception-caught
-            logger.exception("Failed to update runner binary")
+            logger.exception("Failed to reconcile runners.")
             # Failure to reconcile runners is a transient error.
             # The charm automatically reconciles runners on a schedule.
             self.unit.status = MaintenanceStatus(f"Failed to reconcile runners: {err}")
@@ -511,7 +534,6 @@ def _install_deps(self) -> None:
             [
                 "/usr/bin/pip",
                 "install",
-                "flask",
                 "git+https://github.com/canonical/repo-policy-compliance@main",
             ],
             env=env,
@@ -528,6 +550,7 @@ def _install_deps(self) -> None:
                 "cpu-checker",
                 "libvirt-clients",
                 "libvirt-daemon-driver-qemu",
+                "apparmor-utils",
             ],
         )
         execute_command(["/usr/bin/snap", "install", "lxd", "--channel=latest/stable"])
@@ -538,6 +561,28 @@ def _install_deps(self) -> None:
         execute_command(["/snap/bin/lxc", "network", "set", "lxdbr0", "ipv6.address", "none"])
         logger.info("Finished installing charm dependencies.")
 
+        logger.info("Setting apparmor to complain mode.")
+        self._apparmor_complain_mode(
+            [
+                "/etc/apparmor.d/lsb_release",
+                "/etc/apparmor.d/nvidia_modprobe",
+                "/etc/apparmor.d/sbin.dhclient",
+                "/etc/apparmor.d/usr.bin.man",
+                "/etc/apparmor.d/usr.bin.tcpdump",
+                "/etc/apparmor.d/usr.lib.snapd.snap-confine.real",
+                "/etc/apparmor.d/usr.sbin.rsyslogd",
+            ]
+        )
+
+    def _apparmor_complain_mode(self, profiles: list[str]) -> None:
+        """Enable complain mode for the apparmor profile.
+
+        Args:
+            profiles: Profiles to enable complain mode.
+        """
+        for profile in profiles:
+            execute_command(["aa-complain", profile])
+
     @retry(tries=10, delay=15, max_delay=60, backoff=1.5)
     def _start_services(self) -> None:
         """Start services."""

src/errors.py

@@ -35,6 +35,24 @@ class RunnerBinaryError(RunnerError):
     """Error of getting runner binary."""
 
 
+class MissingConfigurationError(Exception):
+    """Error for missing juju configuration.
+
+    Attrs:
+        configs: The missing configurations.
+    """
+
+    def __init__(self, configs: list[str]):
+        """Construct the MissingConfigurationError.
+
+        Args:
+            configs: The missing configurations.
+        """
+        super().__init__(f"Missing required charm configuration: {configs}")
+
+        self.configs = configs
+
+
 class LxdError(Exception):
     """Error for executing LXD actions."""