Integrating repo_policy_compliance to run before runner start job (#63)

* Add repo-policy-compliance as service managed by charm

* Use pre-job script in runners to validation job repo policy.

* Refactor lxc_type.py

pyproject.toml

@@ -11,7 +11,9 @@ skips = ["*/*test.py", "*/test_*.py"]
 branch = true
 omit = [
     # Contains interface for calling LXD. Tested in integration tests and end to end tests.
-    "src/lxd.py"
+    "src/lxd.py",
+    # Contains interface for calling repo policy compliance service. Tested in integration test and end to end tests.
+    "src/repo_policy_compliance_client.py",
 ]
 
 [tool.coverage.report]

requirements.txt

@@ -1,8 +1,8 @@
-typing-extensions
-ops
-requests
-jinja2
 ghapi
+jinja2
+ops
 pylxd @ git+https://github.com/lxc/pylxd
+requests
+typing-extensions
 # Newer version does not work with default OpenSSL version on jammy.
 cryptography <= 38.0.4

src/charm.py

@@ -7,9 +7,14 @@
 
 import functools
 import logging
+import os
+import secrets
+import shutil
 import urllib.error
+from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Dict, Optional, TypeVar
 
+import jinja2
 from ops.charm import (
     ActionEvent,
     CharmBase,
@@ -103,6 +108,11 @@ class GithubRunnerCharm(CharmBase):
 
     _stored = StoredState()
 
+    service_token_path = Path("service_token")
+    repo_check_web_service_path = Path("/home/ubuntu/repo_policy_compliance_service")
+    repo_check_web_service_script = Path("src/repo_policy_compliance_service.py")
+    repo_check_systemd_service = Path("/etc/systemd/system/repo-policy-compliance.service")
+
     def __init__(self, *args, **kargs) -> None:
         """Construct the charm.
 
@@ -128,6 +138,8 @@ def __init__(self, *args, **kargs) -> None:
         if no_proxy := get_env_var("JUJU_CHARM_NO_PROXY"):
             self.proxies["no_proxy"] = no_proxy
 
+        self.service_token = None
+
         self.on.define_event("reconcile_runners", ReconcileRunnersEvent)
         self.on.define_event("update_runner_bin", UpdateRunnerBinEvent)
 
@@ -164,6 +176,9 @@ def _get_runner_manager(
         if not token or not path:
             return None
 
+        if self.service_token is None:
+            self.service_token = self._get_service_token()
+
         if "/" in path:
             paths = path.split("/")
             if len(paths) != 2:
@@ -179,7 +194,7 @@ def _get_runner_manager(
         return RunnerManager(
             app_name,
             unit,
-            RunnerManagerConfig(path, token, "jammy"),
+            RunnerManagerConfig(path, token, "jammy", self.service_token),
             proxies=self.proxies,
         )
 
@@ -193,8 +208,9 @@ def _on_install(self, _event: InstallEvent) -> None:
         self.unit.status = MaintenanceStatus("Installing packages")
 
         try:
-            # The `_install_deps` includes retry.
-            GithubRunnerCharm._install_deps()
+            # The `_start_services`, `_install_deps` includes retry.
+            self._install_deps()
+            self._start_services()
         except SubprocessError as err:
             logger.exception(err)
             # The charm cannot proceed without dependencies.
@@ -236,7 +252,8 @@ def _on_upgrade_charm(self, _event: UpgradeCharmEvent) -> None:
             event: Event of charm upgrade.
         """
         logger.info("Reinstalling dependencies...")
-        GithubRunnerCharm._install_deps()
+        self._install_deps()
+        self._start_services()
 
         logger.info("Flushing the runners...")
         runner_manager = self._get_runner_manager()
@@ -472,13 +489,34 @@ def _reconcile_runners(self, runner_manager: RunnerManager) -> Dict[str, "JsonOb
             self.unit.status = MaintenanceStatus(f"Failed to reconcile runners: {err}")
             return {"delta": {"virtual-machines": 0}}
 
-    @staticmethod
     @retry(tries=10, delay=15, max_delay=60, backoff=1.5)
-    def _install_deps() -> None:
+    def _install_deps(self) -> None:
         """Install dependencies."""
         logger.info("Installing charm dependencies.")
 
         # Binding for snap, apt, and lxd init commands are not available so subprocess.run used.
+        env = {}
+        if "http" in self.proxies:
+            env["HTTP_PROXY"] = self.proxies["http"]
+            env["http_proxy"] = self.proxies["http"]
+        if "https" in self.proxies:
+            env["HTTPS_PROXY"] = self.proxies["https"]
+            env["https_proxy"] = self.proxies["https"]
+        if "no_proxy" in self.proxies:
+            env["NO_PROXY"] = self.proxies["no_proxy"]
+            env["no_proxy"] = self.proxies["no_proxy"]
+
+        execute_command(["/usr/bin/apt-get", "install", "-qy", "gunicorn", "python3-pip"])
+        execute_command(
+            [
+                "/usr/bin/pip",
+                "install",
+                "flask",
+                "git+https://github.com/canonical/repo-policy-compliance@main",
+            ],
+            env=env,
+        )
+
         execute_command(
             ["/usr/bin/apt-get", "remove", "-qy", "lxd", "lxd-client"], check_exit=False
         )
@@ -500,6 +538,58 @@ def _install_deps() -> None:
         execute_command(["/snap/bin/lxc", "network", "set", "lxdbr0", "ipv6.address", "none"])
         logger.info("Finished installing charm dependencies.")
 
+    @retry(tries=10, delay=15, max_delay=60, backoff=1.5)
+    def _start_services(self) -> None:
+        """Start services."""
+        logger.info("Starting charm services...")
+
+        if self.service_token is None:
+            self.service_token = self._get_service_token()
+
+        # Move script to home directory
+        logger.info("Loading the repo policy compliance flask app...")
+        os.makedirs(self.repo_check_web_service_path, exist_ok=True)
+        shutil.copyfile(
+            self.repo_check_web_service_script,
+            self.repo_check_web_service_path / "app.py",
+        )
+
+        # Move the systemd service.
+        logger.info("Loading the repo policy compliance gunicorn systemd service...")
+        environment = jinja2.Environment(
+            loader=jinja2.FileSystemLoader("templates"), autoescape=True
+        )
+
+        service_content = environment.get_template("repo-policy-compliance.service.j2").render(
+            working_directory=str(self.repo_check_web_service_path),
+            charm_token=self.service_token,
+            github_token=self.config["token"],
+            proxies=self.proxies,
+        )
+        self.repo_check_systemd_service.write_text(service_content, encoding="utf-8")
+
+        execute_command(["/usr/bin/systemctl", "start", "repo-policy-compliance"])
+        execute_command(["/usr/bin/systemctl", "enable", "repo-policy-compliance"])
+
+        logger.info("Finished starting charm services")
+
+    def _get_service_token(self) -> str:
+        """Get the service token.
+
+        Returns:
+            The service token.
+        """
+        logger.info("Getting the secret token...")
+        if self.service_token_path.exists():
+            logger.info("Found existing token file.")
+            service_token = self.service_token_path.read_text(encoding="utf-8")
+        else:
+            logger.info("Generate new token.")
+            service_token = secrets.token_hex(16)
+            self.service_token_path.write_text(service_token, encoding="utf-8")
+
+        return service_token
+
 
 if __name__ == "__main__":
     main(GithubRunnerCharm)

src/lxd.py

@@ -8,15 +8,23 @@
 from __future__ import annotations
 
 import io
+import logging
 import tempfile
 from typing import IO, Optional, Tuple, Union
 
 import pylxd.models
 
 from errors import LxdError, SubprocessError
-from lxd_type import LxdInstanceConfig, ResourceProfileConfig, ResourceProfileDevices
+from lxd_type import (
+    LxdInstanceConfig,
+    LxdNetwork,
+    LxdResourceProfileConfig,
+    LxdResourceProfileDevices,
+)
 from utilities import execute_command, secure_run_subprocess
 
+logger = logging.getLogger(__name__)
+
 
 class LxdInstanceFileManager:
     """File manager of a LXD instance.
@@ -67,6 +75,7 @@ def push_file(self, source: str, destination: str, mode: Optional[str] = None) -
         try:
             execute_command(lxc_cmd)
         except SubprocessError as err:
+            logger.exception("Failed to push file")
             raise LxdError(f"Unable to push file into LXD instance {self.instance.name}") from err
 
     def write_file(
@@ -112,6 +121,7 @@ def pull_file(self, source: str, destination: str) -> None:
         try:
             execute_command(lxc_cmd)
         except SubprocessError as err:
+            logger.exception("Failed to pull file")
             raise LxdError(
                 f"Unable to pull file {source} from LXD instance {self.instance.name}"
             ) from err
@@ -177,6 +187,7 @@ def start(self, timeout: int = 30, force: bool = True, wait: bool = False) -> No
         try:
             self._pylxd_instance.start(timeout, force, wait)
         except pylxd.exceptions.LXDAPIException as err:
+            logger.exception("Failed to start LXD instance")
             raise LxdError(f"Unable to start LXD instance {self.name}") from err
 
     def stop(self, timeout: int = 30, force: bool = True, wait: bool = False) -> None:
@@ -193,6 +204,7 @@ def stop(self, timeout: int = 30, force: bool = True, wait: bool = False) -> Non
         try:
             self._pylxd_instance.stop(timeout, force, wait)
         except pylxd.exceptions.LXDAPIException as err:
+            logger.exception("Failed to stop LXD instance")
             raise LxdError(f"Unable to stop LXD instance {self.name}") from err
 
     def delete(self, wait: bool = False) -> None:
@@ -207,6 +219,7 @@ def delete(self, wait: bool = False) -> None:
         try:
             self._pylxd_instance.delete(wait)
         except pylxd.exceptions.LXDAPIException as err:
+            logger.exception("Failed to delete LXD instance")
             raise LxdError(f"Unable to delete LXD instance {self.name}") from err
 
     def execute(self, cmd: list[str], cwd: Optional[str] = None) -> Tuple[int, IO, IO]:
@@ -258,6 +271,7 @@ def all(self) -> list[LxdInstance]:
                 for instance in self._pylxd_client.instances.all()
             ]
         except pylxd.exceptions.LXDAPIException as err:
+            logger.exception("Failed to get all LXD instance")
             raise LxdError("Unable to get all LXD instances") from err
 
     def create(self, config: LxdInstanceConfig, wait: bool) -> LxdInstance:
@@ -277,6 +291,7 @@ def create(self, config: LxdInstanceConfig, wait: bool) -> LxdInstance:
             pylxd_instance = self._pylxd_client.instances.create(config=config, wait=wait)
             return LxdInstance(config["name"], pylxd_instance)
         except pylxd.exceptions.LXDAPIException as err:
+            logger.exception("Failed to create LXD instance")
             raise LxdError(f"Unable to create LXD instance {config['name']}") from err
 
 
@@ -306,10 +321,11 @@ def exists(self, name: str) -> bool:
         try:
             return self._pylxd_client.profiles.exists(name)
         except pylxd.exceptions.LXDAPIException as err:
+            logger.exception("Failed to check if LXD profile exists")
             raise LxdError(f"Unable to check if LXD profile {name} exists") from err
 
     def create(
-        self, name: str, config: ResourceProfileConfig, devices: ResourceProfileDevices
+        self, name: str, config: LxdResourceProfileConfig, devices: LxdResourceProfileDevices
     ) -> None:
         """Create a LXD profile.
 
@@ -324,7 +340,40 @@ def create(
         try:
             self._pylxd_client.profiles.create(name, config, devices)
         except pylxd.exceptions.LXDAPIException as err:
-            raise LxdError(f"Unable to create LXD profile {name} exists") from err
+            logger.exception("Failed to create LXD profile")
+            raise LxdError(f"Unable to create LXD profile {name}") from err
+
+
+# Disable pylint as other method of this class can be extended in the future.
+class LxdNetworkManager:  # pylint: disable=too-few-public-methods
+    """LXD network manager."""
+
+    def __init__(self, pylxd_client: pylxd.Client):
+        """Construct the LXD profile manager.
+
+        Args:
+            pylxd_client: Instance of pylxd.Client.
+        """
+        self._pylxd_client = pylxd_client
+
+    def get(self, name: str) -> LxdNetwork:
+        """Get a LXD network information.
+
+        Args:
+            name: The name of the LXD network.
+
+        Returns:
+            Information on the LXD network.
+        """
+        network = self._pylxd_client.networks.get(name)
+        return LxdNetwork(
+            network.name,
+            network.description,
+            network.type,
+            network.config,
+            network.managed,
+            network.used_by,
+        )
 
 
 # Disable pylint as the public methods of this class in split into instances and profiles.
@@ -336,3 +385,4 @@ def __init__(self):
         pylxd_client = pylxd.Client()
         self.instances = LxdInstanceManager(pylxd_client)
         self.profiles = LxdProfileManager(pylxd_client)
+        self.networks = LxdNetworkManager(pylxd_client)