From 08bef363b9ac413d19098228bf99d856bc947bb3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 29 May 2023 10:44:56 +0800 Subject: [PATCH] Integrating repo_policy_compliance to run before runner start job (#63) * Add repo-policy-compliance as service managed by charm * Use pre-job script in runners to validation job repo policy. * Refactor lxc_type.py --- pyproject.toml | 4 +- requirements.txt | 8 +- src/charm.py | 102 ++++++++++++++++++-- src/lxd.py | 56 ++++++++++- src/lxd_type.py | 42 +++++--- src/repo_policy_compliance_client.py | 48 +++++++++ src/repo_policy_compliance_service.py | 14 +++ src/runner.py | 68 ++++++++----- src/runner_manager.py | 12 +++ src/runner_type.py | 2 + templates/env.j2 | 1 + templates/pre-job.j2 | 11 +++ templates/repo-policy-compliance.service.j2 | 17 ++++ tests/unit/conftest.py | 15 ++- tests/unit/mock.py | 24 +++++ tests/unit/test_charm.py | 24 ++++- tests/unit/test_runner.py | 14 ++- tests/unit/test_runner_manager.py | 2 +- 18 files changed, 401 insertions(+), 63 deletions(-) create mode 100644 src/repo_policy_compliance_client.py create mode 100644 src/repo_policy_compliance_service.py create mode 100644 templates/pre-job.j2 create mode 100644 templates/repo-policy-compliance.service.j2 diff --git a/pyproject.toml b/pyproject.toml index 3f66c0fcf..c72f8d0ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ skips = ["*/*test.py", "*/test_*.py"] branch = true omit = [ # Contains interface for calling LXD. Tested in integration tests and end to end tests. - "src/lxd.py" + "src/lxd.py", + # Contains interface for calling repo policy compliance service. Tested in integration test and end to end tests. + "src/repo_policy_compliance_client.py", ] [tool.coverage.report] diff --git a/requirements.txt b/requirements.txt index d0ec3d098..76bbb1da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -typing-extensions -ops -requests -jinja2 ghapi +jinja2 +ops pylxd @ git+https://github.com/lxc/pylxd +requests +typing-extensions # Newer version does not work with default OpenSSL version on jammy. cryptography <= 38.0.4 diff --git a/src/charm.py b/src/charm.py index 2454c346e..a721674de 100755 --- a/src/charm.py +++ b/src/charm.py @@ -7,9 +7,14 @@ import functools import logging +import os +import secrets +import shutil import urllib.error +from pathlib import Path from typing import TYPE_CHECKING, Callable, Dict, Optional, TypeVar +import jinja2 from ops.charm import ( ActionEvent, CharmBase, @@ -103,6 +108,11 @@ class GithubRunnerCharm(CharmBase): _stored = StoredState() + service_token_path = Path("service_token") + repo_check_web_service_path = Path("/home/ubuntu/repo_policy_compliance_service") + repo_check_web_service_script = Path("src/repo_policy_compliance_service.py") + repo_check_systemd_service = Path("/etc/systemd/system/repo-policy-compliance.service") + def __init__(self, *args, **kargs) -> None: """Construct the charm. @@ -128,6 +138,8 @@ def __init__(self, *args, **kargs) -> None: if no_proxy := get_env_var("JUJU_CHARM_NO_PROXY"): self.proxies["no_proxy"] = no_proxy + self.service_token = None + self.on.define_event("reconcile_runners", ReconcileRunnersEvent) self.on.define_event("update_runner_bin", UpdateRunnerBinEvent) @@ -164,6 +176,9 @@ def _get_runner_manager( if not token or not path: return None + if self.service_token is None: + self.service_token = self._get_service_token() + if "/" in path: paths = path.split("/") if len(paths) != 2: @@ -179,7 +194,7 @@ def _get_runner_manager( return RunnerManager( app_name, unit, - RunnerManagerConfig(path, token, "jammy"), + RunnerManagerConfig(path, token, "jammy", self.service_token), proxies=self.proxies, ) @@ -193,8 +208,9 @@ def _on_install(self, _event: InstallEvent) -> None: self.unit.status = MaintenanceStatus("Installing packages") try: - # The `_install_deps` includes retry. - GithubRunnerCharm._install_deps() + # The `_start_services`, `_install_deps` includes retry. + self._install_deps() + self._start_services() except SubprocessError as err: logger.exception(err) # The charm cannot proceed without dependencies. @@ -236,7 +252,8 @@ def _on_upgrade_charm(self, _event: UpgradeCharmEvent) -> None: event: Event of charm upgrade. """ logger.info("Reinstalling dependencies...") - GithubRunnerCharm._install_deps() + self._install_deps() + self._start_services() logger.info("Flushing the runners...") runner_manager = self._get_runner_manager() @@ -472,13 +489,34 @@ def _reconcile_runners(self, runner_manager: RunnerManager) -> Dict[str, "JsonOb self.unit.status = MaintenanceStatus(f"Failed to reconcile runners: {err}") return {"delta": {"virtual-machines": 0}} - @staticmethod @retry(tries=10, delay=15, max_delay=60, backoff=1.5) - def _install_deps() -> None: + def _install_deps(self) -> None: """Install dependencies.""" logger.info("Installing charm dependencies.") # Binding for snap, apt, and lxd init commands are not available so subprocess.run used. + env = {} + if "http" in self.proxies: + env["HTTP_PROXY"] = self.proxies["http"] + env["http_proxy"] = self.proxies["http"] + if "https" in self.proxies: + env["HTTPS_PROXY"] = self.proxies["https"] + env["https_proxy"] = self.proxies["https"] + if "no_proxy" in self.proxies: + env["NO_PROXY"] = self.proxies["no_proxy"] + env["no_proxy"] = self.proxies["no_proxy"] + + execute_command(["/usr/bin/apt-get", "install", "-qy", "gunicorn", "python3-pip"]) + execute_command( + [ + "/usr/bin/pip", + "install", + "flask", + "git+https://github.com/canonical/repo-policy-compliance@main", + ], + env=env, + ) + execute_command( ["/usr/bin/apt-get", "remove", "-qy", "lxd", "lxd-client"], check_exit=False ) @@ -500,6 +538,58 @@ def _install_deps() -> None: execute_command(["/snap/bin/lxc", "network", "set", "lxdbr0", "ipv6.address", "none"]) logger.info("Finished installing charm dependencies.") + @retry(tries=10, delay=15, max_delay=60, backoff=1.5) + def _start_services(self) -> None: + """Start services.""" + logger.info("Starting charm services...") + + if self.service_token is None: + self.service_token = self._get_service_token() + + # Move script to home directory + logger.info("Loading the repo policy compliance flask app...") + os.makedirs(self.repo_check_web_service_path, exist_ok=True) + shutil.copyfile( + self.repo_check_web_service_script, + self.repo_check_web_service_path / "app.py", + ) + + # Move the systemd service. + logger.info("Loading the repo policy compliance gunicorn systemd service...") + environment = jinja2.Environment( + loader=jinja2.FileSystemLoader("templates"), autoescape=True + ) + + service_content = environment.get_template("repo-policy-compliance.service.j2").render( + working_directory=str(self.repo_check_web_service_path), + charm_token=self.service_token, + github_token=self.config["token"], + proxies=self.proxies, + ) + self.repo_check_systemd_service.write_text(service_content, encoding="utf-8") + + execute_command(["/usr/bin/systemctl", "start", "repo-policy-compliance"]) + execute_command(["/usr/bin/systemctl", "enable", "repo-policy-compliance"]) + + logger.info("Finished starting charm services") + + def _get_service_token(self) -> str: + """Get the service token. + + Returns: + The service token. + """ + logger.info("Getting the secret token...") + if self.service_token_path.exists(): + logger.info("Found existing token file.") + service_token = self.service_token_path.read_text(encoding="utf-8") + else: + logger.info("Generate new token.") + service_token = secrets.token_hex(16) + self.service_token_path.write_text(service_token, encoding="utf-8") + + return service_token + if __name__ == "__main__": main(GithubRunnerCharm) diff --git a/src/lxd.py b/src/lxd.py index da86ea241..4d2aaaa49 100644 --- a/src/lxd.py +++ b/src/lxd.py @@ -8,15 +8,23 @@ from __future__ import annotations import io +import logging import tempfile from typing import IO, Optional, Tuple, Union import pylxd.models from errors import LxdError, SubprocessError -from lxd_type import LxdInstanceConfig, ResourceProfileConfig, ResourceProfileDevices +from lxd_type import ( + LxdInstanceConfig, + LxdNetwork, + LxdResourceProfileConfig, + LxdResourceProfileDevices, +) from utilities import execute_command, secure_run_subprocess +logger = logging.getLogger(__name__) + class LxdInstanceFileManager: """File manager of a LXD instance. @@ -67,6 +75,7 @@ def push_file(self, source: str, destination: str, mode: Optional[str] = None) - try: execute_command(lxc_cmd) except SubprocessError as err: + logger.exception("Failed to push file") raise LxdError(f"Unable to push file into LXD instance {self.instance.name}") from err def write_file( @@ -112,6 +121,7 @@ def pull_file(self, source: str, destination: str) -> None: try: execute_command(lxc_cmd) except SubprocessError as err: + logger.exception("Failed to pull file") raise LxdError( f"Unable to pull file {source} from LXD instance {self.instance.name}" ) from err @@ -177,6 +187,7 @@ def start(self, timeout: int = 30, force: bool = True, wait: bool = False) -> No try: self._pylxd_instance.start(timeout, force, wait) except pylxd.exceptions.LXDAPIException as err: + logger.exception("Failed to start LXD instance") raise LxdError(f"Unable to start LXD instance {self.name}") from err def stop(self, timeout: int = 30, force: bool = True, wait: bool = False) -> None: @@ -193,6 +204,7 @@ def stop(self, timeout: int = 30, force: bool = True, wait: bool = False) -> Non try: self._pylxd_instance.stop(timeout, force, wait) except pylxd.exceptions.LXDAPIException as err: + logger.exception("Failed to stop LXD instance") raise LxdError(f"Unable to stop LXD instance {self.name}") from err def delete(self, wait: bool = False) -> None: @@ -207,6 +219,7 @@ def delete(self, wait: bool = False) -> None: try: self._pylxd_instance.delete(wait) except pylxd.exceptions.LXDAPIException as err: + logger.exception("Failed to delete LXD instance") raise LxdError(f"Unable to delete LXD instance {self.name}") from err def execute(self, cmd: list[str], cwd: Optional[str] = None) -> Tuple[int, IO, IO]: @@ -258,6 +271,7 @@ def all(self) -> list[LxdInstance]: for instance in self._pylxd_client.instances.all() ] except pylxd.exceptions.LXDAPIException as err: + logger.exception("Failed to get all LXD instance") raise LxdError("Unable to get all LXD instances") from err def create(self, config: LxdInstanceConfig, wait: bool) -> LxdInstance: @@ -277,6 +291,7 @@ def create(self, config: LxdInstanceConfig, wait: bool) -> LxdInstance: pylxd_instance = self._pylxd_client.instances.create(config=config, wait=wait) return LxdInstance(config["name"], pylxd_instance) except pylxd.exceptions.LXDAPIException as err: + logger.exception("Failed to create LXD instance") raise LxdError(f"Unable to create LXD instance {config['name']}") from err @@ -306,10 +321,11 @@ def exists(self, name: str) -> bool: try: return self._pylxd_client.profiles.exists(name) except pylxd.exceptions.LXDAPIException as err: + logger.exception("Failed to check if LXD profile exists") raise LxdError(f"Unable to check if LXD profile {name} exists") from err def create( - self, name: str, config: ResourceProfileConfig, devices: ResourceProfileDevices + self, name: str, config: LxdResourceProfileConfig, devices: LxdResourceProfileDevices ) -> None: """Create a LXD profile. @@ -324,7 +340,40 @@ def create( try: self._pylxd_client.profiles.create(name, config, devices) except pylxd.exceptions.LXDAPIException as err: - raise LxdError(f"Unable to create LXD profile {name} exists") from err + logger.exception("Failed to create LXD profile") + raise LxdError(f"Unable to create LXD profile {name}") from err + + +# Disable pylint as other method of this class can be extended in the future. +class LxdNetworkManager: # pylint: disable=too-few-public-methods + """LXD network manager.""" + + def __init__(self, pylxd_client: pylxd.Client): + """Construct the LXD profile manager. + + Args: + pylxd_client: Instance of pylxd.Client. + """ + self._pylxd_client = pylxd_client + + def get(self, name: str) -> LxdNetwork: + """Get a LXD network information. + + Args: + name: The name of the LXD network. + + Returns: + Information on the LXD network. + """ + network = self._pylxd_client.networks.get(name) + return LxdNetwork( + network.name, + network.description, + network.type, + network.config, + network.managed, + network.used_by, + ) # Disable pylint as the public methods of this class in split into instances and profiles. @@ -336,3 +385,4 @@ def __init__(self): pylxd_client = pylxd.Client() self.instances = LxdInstanceManager(pylxd_client) self.profiles = LxdProfileManager(pylxd_client) + self.networks = LxdNetworkManager(pylxd_client) diff --git a/src/lxd_type.py b/src/lxd_type.py index 3491f1866..7e6d0dbc4 100644 --- a/src/lxd_type.py +++ b/src/lxd_type.py @@ -5,17 +5,18 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TypedDict # The keys are not valid identifiers, hence this is defined with function-based syntax. -ResourceProfileConfig = TypedDict( - "ResourceProfileConfig", {"limits.cpu": str, "limits.memory": str} +LxdResourceProfileConfig = TypedDict( + "LxdResourceProfileConfig", {"limits.cpu": str, "limits.memory": str} ) -ResourceProfileConfig.__doc__ = "Represent LXD profile configuration." +LxdResourceProfileConfig.__doc__ = "Configuration LXD profile." -class ResourceProfileDevicesRoot(TypedDict): - """Represents LXD device profile of disk used as root. +class LxdResourceProfileDevicesDisk(TypedDict): + """LXD device profile of disk. The details of the configuration of different types of devices can be found here: https://linuxcontainers.org/lxd/docs/latest/reference/devices/ @@ -33,16 +34,7 @@ class ResourceProfileDevicesRoot(TypedDict): size: str -class ResourceProfileDevices(TypedDict): - """Represents LXD device profile for a LXD instance. - - A device for root is defined. - - The details of the configuration can be found in this link: - https://linuxcontainers.org/lxd/docs/latest/reference/devices/ - """ - - root: ResourceProfileDevicesRoot +LxdResourceProfileDevices = dict[str, LxdResourceProfileDevicesDisk] class LxdInstanceConfigSource(TypedDict): @@ -63,3 +55,23 @@ class LxdInstanceConfig(TypedDict): source: LxdInstanceConfigSource ephemeral: bool profiles: list[str] + + +# The keys are not valid identifiers, hence this is defined with function-based syntax. +LxdNetworkConfig = TypedDict( + "LxdNetworkConfig", + {"ipv4.address": str, "ipv4.nat": str, "ipv6.address": str, "ipv6.nat": str}, +) +LxdNetworkConfig.__doc__ = "Represent LXD network configuration." + + +@dataclass +class LxdNetwork: + """LXD network information.""" + + name: str + description: str + type: str + config: LxdNetworkConfig + managed: bool + used_by: tuple[str] diff --git a/src/repo_policy_compliance_client.py b/src/repo_policy_compliance_client.py new file mode 100644 index 000000000..2c3318983 --- /dev/null +++ b/src/repo_policy_compliance_client.py @@ -0,0 +1,48 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Client for requesting repo policy compliance service.""" + +import logging +from urllib.parse import urljoin + +import requests + +logger = logging.getLogger(__name__) + + +# Disable pylint as other method of this class can be extended in the future. +class RepoPolicyComplianceClient: # pylint: disable=too-few-public-methods + """Client for repo policy compliance service. + + Attrs: + base_url: Base url to the repo policy compliance service. + token: Charm token configured for the repo policy compliance service. + """ + + def __init__(self, session: requests.Session, url: str, charm_token: str) -> None: + """Construct the RepoPolicyComplianceClient. + + Args: + session: The request Session object for making HTTP requests. + url: Base URL to the repo policy compliance service. + charm_token: Charm token configured for the repo policy compliance service. + """ + self._session = session + self.base_url = url + self.token = charm_token + + def get_one_time_token(self) -> str: + """Get a single-use token for repo policy compliance check. + + Returns: + The one-time token to be used in a single request of repo policy compliance check. + """ + url = urljoin(self.base_url, "one-time-token") + try: + response = self._session.get(url, headers={"Authorization": f"Bearer {self.token}"}) + response.raise_for_status() + return response.content.decode("utf-8") + except requests.HTTPError: + logger.exception("Unable to get one time token from repo policy compliance service.") + raise diff --git a/src/repo_policy_compliance_service.py b/src/repo_policy_compliance_service.py new file mode 100644 index 000000000..b9c896098 --- /dev/null +++ b/src/repo_policy_compliance_service.py @@ -0,0 +1,14 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Flask application for repo policy compliance. + +This module is loaded into juju unit and run with gunicorn. +""" + +# This module is executed in a different environment. +from flask import Flask # pylint: disable=import-error +from repo_policy_compliance.blueprint import repo_policy_compliance # pylint: disable=import-error + +app = Flask(__name__) +app.register_blueprint(repo_policy_compliance) diff --git a/src/runner.py b/src/runner.py index 402b28434..94b2c3ff5 100644 --- a/src/runner.py +++ b/src/runner.py @@ -51,6 +51,7 @@ class Runner: env_file = runner_application / ".env" config_script = runner_application / "config.sh" runner_script = runner_application / "start.sh" + pre_job_script = runner_application / "pre-job.sh" def __init__( self, @@ -155,32 +156,34 @@ def remove(self, remove_token: str) -> None: except LxdError as err: raise RunnerRemoveError(f"Unable to remove {self.config.name}") from err + if self.status.runner_id is None: + return + # The runner should cleanup itself. Cleanup on GitHub in case of runner cleanup error. - if self.status.runner_id is not None: - if isinstance(self.config.path, GitHubRepo): - logger.debug( - "Ensure runner %s with id %s is removed from GitHub repo %s/%s", - self.config.name, - self.status.runner_id, - self.config.path.owner, - self.config.path.repo, - ) - self._clients.github.actions.delete_self_hosted_runner_from_repo( - owner=self.config.path.owner, - repo=self.config.path.repo, - runner_id=self.status.runner_id, - ) - if isinstance(self.config.path, GitHubOrg): - logger.debug( - "Ensure runner %s with id %s is removed from GitHub org %s", - self.config.name, - self.status.runner_id, - self.config.path.org, - ) - self._clients.github.actions.delete_self_hosted_runner_from_org( - org=self.config.path.org, - runner_id=self.status.runner_id, - ) + if isinstance(self.config.path, GitHubRepo): + logger.debug( + "Ensure runner %s with id %s is removed from GitHub repo %s/%s", + self.config.name, + self.status.runner_id, + self.config.path.owner, + self.config.path.repo, + ) + self._clients.github.actions.delete_self_hosted_runner_from_repo( + owner=self.config.path.owner, + repo=self.config.path.repo, + runner_id=self.status.runner_id, + ) + if isinstance(self.config.path, GitHubOrg): + logger.debug( + "Ensure runner %s with id %s is removed from GitHub org %s", + self.config.name, + self.status.runner_id, + self.config.path.org, + ) + self._clients.github.actions.delete_self_hosted_runner_from_org( + org=self.config.path.org, + runner_id=self.status.runner_id, + ) @retry(tries=5, delay=1, local_logger=logger) def _create_instance( @@ -382,6 +385,19 @@ def _configure_runner(self) -> None: self.instance.execute(["/usr/bin/sudo", "chown", "ubuntu:ubuntu", str(self.runner_script)]) self.instance.execute(["/usr/bin/sudo", "chmod", "u+x", str(self.runner_script)]) + # Load the runner pre-job script. + bridge_address_range = self._clients.lxd.networks.get("lxdbr0").config["ipv4.address"] + host_ip, _ = bridge_address_range.split("/") + one_time_token = self._clients.repo.get_one_time_token() + pre_job_contents = self._clients.jinja.get_template("pre-job.j2").render( + host_ip=host_ip, one_time_token=one_time_token + ) + self._put_file(str(self.pre_job_script), pre_job_contents) + self.instance.execute( + ["/usr/bin/sudo", "chown", "ubuntu:ubuntu", str(self.pre_job_script)] + ) + self.instance.execute(["/usr/bin/sudo", "chmod", "u+x", str(self.pre_job_script)]) + # Set permission to the same as GitHub-hosted runner for this directory. # Some GitHub Actions require this permission setting to run. # As the user already has sudo access, this does not give the user any additional access. @@ -395,7 +411,7 @@ def _configure_runner(self) -> None: # Load `.env` config file for GitHub self-hosted runner. env_contents = self._clients.jinja.get_template("env.j2").render( - proxies=self.config.proxies + proxies=self.config.proxies, pre_job_script=str(self.pre_job_script) ) self._put_file(str(self.env_file), env_contents) self.instance.execute(["/usr/bin/chown", "ubuntu:ubuntu", str(self.env_file)]) diff --git a/src/runner_manager.py b/src/runner_manager.py index 6c8e86f3c..eebe7dddc 100644 --- a/src/runner_manager.py +++ b/src/runner_manager.py @@ -32,6 +32,7 @@ SelfHostedRunner, ) from lxd import LxdClient, LxdInstance +from repo_policy_compliance_client import RepoPolicyComplianceClient from runner import Runner, RunnerClients, RunnerConfig, RunnerStatus from runner_type import GitHubOrg, GitHubPath, GitHubRepo, ProxySetting, VirtualMachineResources from utilities import retry, set_env_var @@ -53,6 +54,7 @@ class RunnerManagerConfig: path: GitHubPath token: str image: str + service_token: str @dataclass @@ -115,10 +117,20 @@ def __init__( opener = urllib.request.build_opener(proxy) fastcore.net._opener = opener + # The repo policy compliance service is on localhost and should not have any proxies + # setting configured. The is a separated requests Session as the other one configured + # according proxies setting provided by user. + local_session = requests.Session() + local_session.mount("http://", adapter) + local_session.mount("https://", adapter) + self._clients = RunnerClients( GhApi(token=self.config.token), jinja2.Environment(loader=jinja2.FileSystemLoader("templates"), autoescape=True), LxdClient(), + RepoPolicyComplianceClient( + local_session, "http://127.0.0.1:8080", self.config.service_token + ), ) @retry(tries=5, delay=30, local_logger=logger) diff --git a/src/runner_type.py b/src/runner_type.py index 495e78357..83530a5b2 100644 --- a/src/runner_type.py +++ b/src/runner_type.py @@ -11,6 +11,7 @@ from ghapi.all import GhApi from lxd import LxdClient +from repo_policy_compliance_client import RepoPolicyComplianceClient class ProxySetting(TypedDict, total=False): @@ -69,6 +70,7 @@ class RunnerClients: github: GhApi jinja: jinja2.Environment lxd: LxdClient + repo: RepoPolicyComplianceClient @dataclass diff --git a/templates/env.j2 b/templates/env.j2 index 2649415b2..8e99da4b4 100644 --- a/templates/env.j2 +++ b/templates/env.j2 @@ -6,3 +6,4 @@ http_proxy={{proxies['http']}} https_proxy={{proxies['https']}} no_proxy={{proxies['no_proxy']}} LANG=C.UTF-8 +ACTIONS_RUNNER_HOOK_JOB_STARTED={{pre_job_script}} diff --git a/templates/pre-job.j2 b/templates/pre-job.j2 new file mode 100644 index 000000000..968a77dbb --- /dev/null +++ b/templates/pre-job.j2 @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +GITHUB_SOURCE_REPOSITORY=$(cat "${GITHUB_EVENT_PATH}" | jq -r '.pull_request.head.repo.full_name') + +# Request repo-policy-compliance service check. +curl --noproxy '*' \ + --fail-with-body \ + -H 'Authorization: Bearer {{one_time_token}}' \ + -H 'Content-Type: application/json' \ + -d "{\"repository_name\": \"${GITHUB_REPOSITORY}\", \"source_repository_name\": \"${GITHUB_SOURCE_REPOSITORY}\", \"target_branch_name\": \"${GITHUB_BASE_REF}\", \"source_branch_name\": \"${GITHUB_HEAD_REF}\", \"commit_sha\": \"${GITHUB_SHA}\"}" \ + http://{{host_ip}}:8080/check-run diff --git a/templates/repo-policy-compliance.service.j2 b/templates/repo-policy-compliance.service.j2 new file mode 100644 index 000000000..afde32b2e --- /dev/null +++ b/templates/repo-policy-compliance.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Gunicorn instance to serve repo policy compliance endpoints +After=network.target + +[Service] +User=ubuntu +Group=www-data +WorkingDirectory={{working_directory}} +Environment="GITHUB_TOKEN={{github_token}}" +Environment="CHARM_TOKEN={{charm_token}}" +Environment="HTTP_PROXY={{proxies['http']}}" +Environment="HTTPS_PROXY={{proxies['https']}}" +Environment="NO_PROXY={{proxies['no_proxy']}}" +Environment="http_proxy={{proxies['http']}}" +Environment="https_proxy={{proxies['https']}}" +Environment="no_proxy={{proxies['no_proxy']}}" +ExecStart=/usr/bin/gunicorn --bind 0.0.0.0:8080 app:app diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5cdc37bbf..20bcefc73 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,16 +2,29 @@ # See LICENSE file for licensing details. import unittest.mock +from pathlib import Path import pytest -from tests.unit.mock import MockGhapiClient, MockLxdClient +from tests.unit.mock import MockGhapiClient, MockLxdClient, MockRepoPolicyComplianceClient @pytest.fixture(autouse=True) def mocks(monkeypatch, tmp_path): + monkeypatch.setattr( + "charm.GithubRunnerCharm.service_token_path", Path(tmp_path / "mock_service_token") + ) + monkeypatch.setattr( + "charm.GithubRunnerCharm.repo_check_systemd_service", Path(tmp_path / "systemd_service") + ) + monkeypatch.setattr("charm.os", unittest.mock.MagicMock()) + monkeypatch.setattr("charm.shutil", unittest.mock.MagicMock()) + monkeypatch.setattr("charm.jinja2", unittest.mock.MagicMock()) monkeypatch.setattr("runner.time", unittest.mock.MagicMock()) monkeypatch.setattr("runner_manager.GhApi", MockGhapiClient) monkeypatch.setattr("runner_manager.jinja2", unittest.mock.MagicMock()) monkeypatch.setattr("runner_manager.LxdClient", MockLxdClient) + monkeypatch.setattr( + "runner_manager.RepoPolicyComplianceClient", MockRepoPolicyComplianceClient + ) monkeypatch.setattr("utilities.time", unittest.mock.MagicMock()) diff --git a/tests/unit/mock.py b/tests/unit/mock.py index 29f14cd0a..1238c69e8 100644 --- a/tests/unit/mock.py +++ b/tests/unit/mock.py @@ -12,6 +12,7 @@ from errors import LxdError, RunnerError from github_type import RegistrationToken, RemoveToken, RunnerApplication +from lxd_type import LxdNetwork from runner import LxdInstanceConfig logger = logging.getLogger(__name__) @@ -34,6 +35,7 @@ class MockLxdClient: def __init__(self): self.instances = MockLxdInstanceManager() self.profiles = MockLxdProfileManager() + self.networks = MockLxdNetworkManager() class MockLxdInstanceManager: @@ -66,6 +68,18 @@ def exists(self, name) -> bool: return name in self.profiles +class MockLxdNetworkManager: + """Mock the behavior of the lxd networks""" + + def __init__(self): + pass + + def get(self, name: str) -> LxdNetwork: + return LxdNetwork( + "lxdbr0", "", "bridge", {"ipv4.address": "10.1.1.1/24"}, True, ("default") + ) + + class MockLxdInstance: """Mock the behavior of a lxd Instance.""" @@ -198,3 +212,13 @@ def delete_self_hosted_runner_from_repo(self, owner: str, repo: str, runner_id: def delete_self_hosted_runner_from_org(self, org: str, runner_id: str): pass + + +class MockRepoPolicyComplianceClient: + """Mock for RepoPolicyComplianceClient.""" + + def __init__(self, session=None, url=None, charm_token=None): + pass + + def get_one_time_token(self) -> str: + return "MOCK_TOKEN_" + secrets.token_hex(8) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 7d68fb3da..74de4924b 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -103,11 +103,15 @@ def test_org_register(self, run, wt, rm): ) harness.begin() harness.charm.on.config_changed.emit() + token = harness.charm.service_token rm.assert_called_with( "github-runner", "0", RunnerManagerConfig( - path=GitHubOrg(org="mockorg", group="mockgroup"), token="mocktoken", image="jammy" + path=GitHubOrg(org="mockorg", group="mockgroup"), + token="mocktoken", + image="jammy", + service_token=token, ), proxies={}, ) @@ -122,11 +126,15 @@ def test_repo_register(self, run, wt, rm): ) harness.begin() harness.charm.on.config_changed.emit() + token = harness.charm.service_token rm.assert_called_with( "github-runner", "0", RunnerManagerConfig( - path=GitHubRepo(owner="mockorg", repo="repo"), token="mocktoken", image="jammy" + path=GitHubRepo(owner="mockorg", repo="repo"), + token="mocktoken", + image="jammy", + service_token=token, ), proxies={}, ) @@ -143,11 +151,15 @@ def test_update_config(self, run, wt, rm): # update to 0 virtual machines harness.update_config({"virtual-machines": 0}) harness.charm.on.reconcile_runners.emit() + token = harness.charm.service_token rm.assert_called_with( "github-runner", "0", RunnerManagerConfig( - path=GitHubRepo(owner="mockorg", repo="repo"), token="mocktoken", image="jammy" + path=GitHubRepo(owner="mockorg", repo="repo"), + token="mocktoken", + image="jammy", + service_token=token, ), proxies={}, ) @@ -157,11 +169,15 @@ def test_update_config(self, run, wt, rm): # update to 10 VMs with 4 cpu and 7GiB memory harness.update_config({"virtual-machines": 10, "vm-cpu": 4}) harness.charm.on.reconcile_runners.emit() + token = harness.charm.service_token rm.assert_called_with( "github-runner", "0", RunnerManagerConfig( - path=GitHubRepo(owner="mockorg", repo="repo"), token="mocktoken", image="jammy" + path=GitHubRepo(owner="mockorg", repo="repo"), + token="mocktoken", + image="jammy", + service_token=token, ), proxies={}, ) diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py index c1011ab4a..32e89811a 100644 --- a/tests/unit/test_runner.py +++ b/tests/unit/test_runner.py @@ -13,7 +13,12 @@ from errors import RunnerCreateError from runner import Runner, RunnerClients, RunnerConfig, RunnerStatus from runner_type import GitHubOrg, GitHubRepo, VirtualMachineResources -from tests.unit.mock import MockLxdClient, mock_lxd_error_func, mock_runner_error_func +from tests.unit.mock import ( + MockLxdClient, + MockRepoPolicyComplianceClient, + mock_lxd_error_func, + mock_runner_error_func, +) @pytest.fixture(scope="module", name="vm_resources") @@ -58,7 +63,12 @@ def mock_lxd_client_fixture(): ], ) def runner_fixture(request, lxd: MockLxdClient): - client = RunnerClients(MagicMock(), MagicMock(), lxd) + client = RunnerClients( + MagicMock(), + MagicMock(), + lxd, + MockRepoPolicyComplianceClient(), + ) config = RunnerConfig("test_app", request.param[0], request.param[1], "test_runner") status = RunnerStatus() return Runner( diff --git a/tests/unit/test_runner_manager.py b/tests/unit/test_runner_manager.py index 1dbf58b31..31fac9d89 100644 --- a/tests/unit/test_runner_manager.py +++ b/tests/unit/test_runner_manager.py @@ -39,7 +39,7 @@ def runner_manager_fixture(request, tmp_path, monkeypatch, token): runner_manager = RunnerManager( "test app", "0", - RunnerManagerConfig(request.param[0], token, "jammy"), + RunnerManagerConfig(request.param[0], token, "jammy", secrets.token_hex(16)), proxies=request.param[1], ) return runner_manager