Merge branch 'main' into auto-kernel-upgrade

.github/workflows/e2e_test.yaml

@@ -100,6 +100,9 @@ jobs:
         with:
           name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm
 
+      - name: Enable br_netfilter
+        run: sudo modprobe br_netfilter
+
       - name: Deploy github-runner Charm
         run: |
           cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm
@@ -109,6 +112,7 @@ jobs:
             --config path=${{ secrets.E2E_TESTING_REPO }} \
             --config token=${{ secrets.E2E_TESTING_TOKEN }} \
             --config virtual-machines=1 \
+            --config denylist=10.0.0.0/8 \
             --config test-mode=insecure
 
       - name: Watch github-runner
@@ -136,6 +140,10 @@ jobs:
             exit 1
           fi
 
+      - name: Show Firewall Rules
+        run: |
+          juju ssh ${{ needs.run-id.outputs.run-id }}/0 sudo nft list ruleset
+
   e2e-test:
     name: End-to-End Test
     needs: [ build-charm, run-id ]
@@ -190,3 +198,7 @@ jobs:
       # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH.
       - name: test check-jsonschema
         run: check-jsonschema --version
+      - name: Test Firewall
+        run: |
+          HOST_IP=$(ip route | grep default | cut -f 3 -d" ")
+          [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ]

config.yaml

@@ -58,3 +58,10 @@ options:
     description: >
       When set to 'insecure', the charm test mode is activated, which may deactivate some security
       hardening measures.
+  denylist:
+    type: string
+    default: ""
+    description: >
+      A comma separated list of IPv4 networks in CIDR notation that runners can not access.
+      The runner will always have access to essential services such as DHCP and DNS regardless 
+      of the denylist configuration.

src/charm.py

@@ -29,6 +29,7 @@
 
 from errors import MissingConfigurationError, RunnerBinaryError, RunnerError, SubprocessError
 from event_timer import EventTimer, TimerDisableError, TimerEnableError
+from firewall import Firewall, FirewallEntry
 from github_type import GitHubRunnerStatus
 from runner import LXD_PROFILE_YAML
 from runner_manager import RunnerManager, RunnerManagerConfig
@@ -162,12 +163,16 @@ def __init__(self, *args, **kargs) -> None:
         self.framework.observe(self.on.flush_runners_action, self._on_flush_runners_action)
         self.framework.observe(self.on.update_runner_bin_action, self._on_update_runner_bin)
 
+    @retry(tries=5, delay=15, max_delay=60, backoff=1.5, local_logger=logger)
     def _create_memory_storage(self, path: Path, size: int) -> None:
         """Create a tmpfs-based LVM volume group.
 
         Args:
             path: Path to directory for memory storage.
             size: Size of the tmpfs in kilobytes.
+
+        Raises:
+            RunnerError: Unable to setup storage for runner.
         """
         try:
             # Create tmpfs if not exists, else resize it.
@@ -178,14 +183,30 @@ def _create_memory_storage(self, path: Path, size: int) -> None:
                 )
             else:
                 execute_command(["mount", "-o", f"remount,size={size}k", str(path)])
-        except OSError as err:
-            logger.exception("Unable to create directory")
-            raise RunnerError("Problem with runner storage due to unable setup directory") from err
-        except SubprocessError as err:
-            logger.exception("Unable to create or resize tmpfs")
-            raise RunnerError(
-                "Problem with runner storage due to unable to create or resize tmpfs"
-            ) from err
+        except (OSError, SubprocessError) as err:
+            logger.exception("Unable to setup storage directory")
+            # Remove the path if is not in use. If the tmpfs is in use, the removal will fail.
+            if path.exists():
+                shutil.rmtree(path, ignore_errors=True)
+                path.rmdir()
+                logger.info("Cleaned up storage directory")
+            raise RunnerError("Failed to configure runner storage") from err
+
+    @retry(tries=10, delay=15, max_delay=60, backoff=1.5, local_logger=logger)
+    def _ensure_service_health(self) -> None:
+        """Ensure services managed by the charm is healthy.
+
+        Services managed include:
+         * repo-policy-compliance
+        """
+        logger.info("Checking health of repo-policy-compliance service")
+        try:
+            execute_command(["/usr/bin/systemctl", "is-active", "repo-policy-compliance"])
+        except SubprocessError:
+            logger.exception("Found inactive repo-policy-compliance service")
+            execute_command(["/usr/bin/systemctl", "restart", "repo-policy-compliance"])
+            logger.exception("Restart repo-policy-compliance service")
+            raise
 
     def _get_runner_manager(
         self, token: Optional[str] = None, path: Optional[str] = None
@@ -200,6 +221,8 @@ def _get_runner_manager(
         Returns:
             An instance of RunnerManager.
         """
+        self._ensure_service_health()
+
         if token is None:
             token = self.config["token"]
         if path is None:
@@ -259,7 +282,7 @@ def _on_install(self, _event: InstallEvent) -> None:
             # The charm cannot proceed without dependencies.
             self.unit.status = BlockedStatus("Failed to install dependencies")
             return
-
+        self._refresh_firewall()
         runner_manager = self._get_runner_manager()
         if runner_manager:
             self.unit.status = MaintenanceStatus("Downloading runner binary")
@@ -307,6 +330,7 @@ def _on_upgrade_charm(self, _event: UpgradeCharmEvent) -> None:
         logger.info("Reinstalling dependencies...")
         self._install_deps()
         self._start_services()
+        self._refresh_firewall()
 
         logger.info("Flushing the runners...")
         runner_manager = self._get_runner_manager()
@@ -323,6 +347,7 @@ def _on_config_changed(self, _event: ConfigChangedEvent) -> None:
         Args:
             event: Event of configuration change.
         """
+        self._refresh_firewall()
         try:
             self._event_timer.ensure_event_timer(
                 "update-runner-bin", self.config["update-interval"]
@@ -542,7 +567,7 @@ def _reconcile_runners(self, runner_manager: RunnerManager) -> Dict[str, "JsonOb
             self.unit.status = MaintenanceStatus(f"Failed to reconcile runners: {err}")
             return {"delta": {"virtual-machines": 0}}
 
-    @retry(tries=10, delay=15, max_delay=60, backoff=1.5)
+    @retry(tries=10, delay=15, max_delay=60, backoff=1.5, local_logger=logger)
     def _install_deps(self) -> None:
         """Install dependencies."""
         logger.info("Installing charm dependencies.")
@@ -560,7 +585,10 @@ def _install_deps(self) -> None:
             env["no_proxy"] = self.proxies["no_proxy"]
 
         execute_command(["/usr/bin/apt-get", "update"])
-        execute_command(["/usr/bin/apt-get", "install", "-qy", "gunicorn", "python3-pip"])
+        # install dependencies used by repo-policy-compliance and the firewall
+        execute_command(
+            ["/usr/bin/apt-get", "install", "-qy", "gunicorn", "python3-pip", "nftables"]
+        )
         execute_command(
             [
                 "/usr/bin/pip",
@@ -588,11 +616,26 @@ def _install_deps(self) -> None:
         execute_command(["/usr/bin/snap", "refresh", "lxd", "--channel=latest/stable"])
         execute_command(["/snap/bin/lxd", "waitready"])
         execute_command(["/snap/bin/lxd", "init", "--auto"])
-        execute_command(["/usr/bin/chmod", "a+wr", "/var/snap/lxd/common/lxd/unix.socket"])
         execute_command(["/snap/bin/lxc", "network", "set", "lxdbr0", "ipv6.address", "none"])
+        if not LXD_PROFILE_YAML.exists():
+            execute_command(["/usr/sbin/modprobe", "br_netfilter"])
+        execute_command(
+            [
+                "/snap/bin/lxc",
+                "profile",
+                "device",
+                "set",
+                "default",
+                "eth0",
+                "security.ipv4_filtering=true",
+                "security.ipv6_filtering=true",
+                "security.mac_filtering=true",
+                "security.port_isolation=true",
+            ]
+        )
         logger.info("Finished installing charm dependencies.")
 
-    @retry(tries=10, delay=15, max_delay=60, backoff=1.5)
+    @retry(tries=10, delay=15, max_delay=60, backoff=1.5, local_logger=logger)
     def _start_services(self) -> None:
         """Start services."""
         logger.info("Starting charm services...")
@@ -644,6 +687,22 @@ def _get_service_token(self) -> str:
 
         return service_token
 
+    def _refresh_firewall(self):
+        """Refresh the firewall configuration and rules."""
+        firewall_denylist_config = self.config.get("denylist")
+        denylist = []
+        if firewall_denylist_config.strip():
+            denylist = [
+                FirewallEntry.decode(entry.strip())
+                for entry in firewall_denylist_config.split(",")
+            ]
+        firewall = Firewall("lxdbr0")
+        firewall.refresh_firewall(denylist)
+        logger.debug(
+            "firewall update, current firewall: %s",
+            execute_command(["/usr/sbin/nft", "list", "ruleset"]),
+        )
+
 
 if __name__ == "__main__":
     main(GithubRunnerCharm)

src/firewall.py

@@ -0,0 +1,152 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""The runner firewall manager."""
+import dataclasses
+import ipaddress
+import json
+import typing
+
+import yaml
+
+from utilities import execute_command
+
+
+@dataclasses.dataclass
+class FirewallEntry:
+    """Represent an entry in the firewall.
+
+    Attrs:
+        ip_range: The IP address range using CIDR notation.
+    """
+
+    ip_range: str
+
+    @classmethod
+    def decode(cls, entry: str) -> "FirewallEntry":
+        """Decode a firewall entry from a string.
+
+        Args:
+            entry: The firewall entry string, e.g. '192.168.0.1:80' or '192.168.0.0/24:80-90:udp'.
+
+        Returns:
+            FirewallEntry: A FirewallEntry instance representing the decoded entry.
+
+        Raises:
+            ValueError: If the entry string is not in the expected format.
+        """
+        try:
+            ipaddress.IPv4Network(entry)
+        except ValueError as exc:
+            raise ValueError(f"incorrect firewall entry format: {entry}") from exc
+        return cls(ip_range=entry)
+
+
+class Firewall:  # pylint: disable=too-few-public-methods
+    """Represent a firewall and provides methods to refresh its configuration."""
+
+    _ACL_RULESET_NAME = "github"
+
+    def __init__(self, network: str):
+        """Initialize a new Firewall instance.
+
+        Args:
+            network: The LXD network name.
+        """
+        self._network = network
+
+    def get_host_ip(self) -> str:
+        """Get the host IP address for the corresponding LXD network.
+
+        Returns:
+            The host IP address.
+        """
+        address = execute_command(
+            ["/snap/bin/lxc", "network", "get", self._network, "ipv4.address"]
+        )
+        return str(ipaddress.IPv4Interface(address.strip()).ip)
+
+    def refresh_firewall(self, denylist: typing.List[FirewallEntry]):
+        """Refresh the firewall configuration.
+
+        Args:
+            denylist: The list of FirewallEntry objects to allow.
+        """
+        current_acls = [
+            acl["name"]
+            for acl in yaml.safe_load(
+                execute_command(["lxc", "network", "acl", "list", "-f", "yaml"])
+            )
+        ]
+        if self._ACL_RULESET_NAME not in current_acls:
+            execute_command(["/snap/bin/lxc", "network", "acl", "create", self._ACL_RULESET_NAME])
+        execute_command(
+            [
+                "/snap/bin/lxc",
+                "network",
+                "set",
+                self._network,
+                f"security.acls={self._ACL_RULESET_NAME}",
+            ]
+        )
+        execute_command(
+            [
+                "/snap/bin/lxc",
+                "network",
+                "set",
+                self._network,
+                "security.acls.default.egress.action=allow",
+            ]
+        )
+        acl_config = yaml.safe_load(
+            execute_command(["/snap/bin/lxc", "network", "acl", "show", self._ACL_RULESET_NAME])
+        )
+        host_ip = self.get_host_ip()
+        egress_rules = [
+            {
+                "action": "reject",
+                "destination": host_ip,
+                "destination_port": "1-8079,8081-65535",
+                "protocol": "tcp",
+                "state": "enabled",
+            },
+            {
+                "action": "reject",
+                "destination": host_ip,
+                "protocol": "udp",
+                "state": "enabled",
+            },
+            {
+                "action": "reject",
+                "destination": host_ip,
+                "protocol": "icmp4",
+                "state": "enabled",
+            },
+            {
+                "action": "reject",
+                "destination": "::/0",
+                "state": "enabled",
+            },
+        ]
+        host_network = ipaddress.IPv4Network(host_ip)
+        for entry in denylist:
+            entry_network = ipaddress.IPv4Network(entry.ip_range)
+            try:
+                excluded = list(entry_network.address_exclude(host_network))
+            except ValueError:
+                excluded = [entry_network]
+            egress_rules.extend(
+                [
+                    {
+                        "action": "reject",
+                        "destination": str(ip),
+                        "state": "enabled",
+                    }
+                    for ip in excluded
+                ]
+            )
+        acl_config["egress"] = egress_rules
+        execute_command(
+            ["lxc", "network", "acl", "edit", self._ACL_RULESET_NAME],
+            input=json.dumps(acl_config).encode("ascii"),
+        )

src/runner.py

@@ -409,7 +409,7 @@ def _install_binary(self, binary: Path) -> None:
             )
             raise RunnerFileLoadError(f"Failed to load runner binary on {self.config.name}")
 
-    @retry(tries=5, delay=1, local_logger=logger)
+    @retry(tries=5, delay=10, max_delay=60, backoff=2, local_logger=logger)
     def _configure_runner(self) -> None:
         """Load configuration on to the runner.
 

src/runner_manager.py

@@ -108,7 +108,7 @@ def __init__(
         self.session = requests.Session()
         adapter = requests.adapters.HTTPAdapter(
             max_retries=urllib3.Retry(
-                total=10, backoff_factor=0.3, status_forcelist=[500, 502, 503, 504]
+                total=3, backoff_factor=0.3, status_forcelist=[500, 502, 503, 504]
             )
         )
         self.session.mount("http://", adapter)