Merge pull request #2026 from Chris-Peterson444/fake-ubuntu-drivers-new

ubuntu-drivers debug wrapper

scripts/kvm-test.py

@@ -23,8 +23,8 @@
 import sys
 import tempfile
 from typing import List, Optional, Tuple
-import yaml
 
+import yaml
 
 cfg = '''
 iso:
@@ -240,6 +240,11 @@ def load_config(self):
                     package)''')
 parser.add_argument('--profile', default="server",
                     help='load predefined memory, disk size and qemu options')
+parser.add_argument('--kernel-cmdline', action='append', default=[],
+                    dest='kernel_appends',
+                    help=('Use to append argument(s) to kernel command line.'
+                          'Can be passed repeatedly.'),
+                    )
 
 
 cc_group = parser.add_mutually_exclusive_group()
@@ -555,7 +560,7 @@ def install(ctx):
     with tempfile.TemporaryDirectory() as tempdir:
         mntdir = f'{tempdir}/mnt'
         os.mkdir(mntdir)
-        appends = []
+        appends = ctx.args.kernel_appends
 
         with kvm_prepare_common(ctx) as kvm:
 

scripts/slimy-update-snap.sh

@@ -58,8 +58,10 @@ rm -rf new/lib/python3.10/site-packages/curtin
 
 if [ -d new/lib/python3.10/site-packages/subiquity ] ; then
     subiquity_dest=new/lib/python3.10/site-packages
+    bin_dest=new/system_scripts/
 elif [ -d new/bin/subiquity/subiquity ] ; then
     subiquity_dest=new/bin/subiquity
+    bin_dest=new/bin/subiquity/system_scripts/
 else
     echo "unrecognized snap" >&2
     exit 1
@@ -72,5 +74,6 @@ rm -rf "${subiquity_dest}/subiquitycore"
 
 rsync -a --chown 0:0 $src/curtin/curtin new/lib/python3.10/site-packages
 rsync -a --chown 0:0 $src/subiquity $src/subiquitycore $subiquity_dest
+rsync -a --chown 0:0 $src/system_scripts/ $bin_dest
 
 snapcraft pack new --output $new

setup.py

@@ -124,6 +124,7 @@ class build(distutils.command.build.build):
           'bin/subiquity-service',
           'bin/subiquity-server',
           'bin/subiquity-cmd',
+          'system_scripts/subiquity-umockdev-wrapper',
       ],
       entry_points={
           'console_scripts': [

snapcraft.yaml

@@ -44,8 +44,11 @@ apps:
     environment:
       PYTHONIOENCODING: utf-8
       SUBIQUITY_ROOT: $SNAP
+      PYTHONPATH_ORIG: $PYTHONPATH
+      PYTHON_ORIG: $PYTHON
       PYTHON: $SNAP/usr/bin/python3.10
       PY3OR2_PYTHON: $SNAP/usr/bin/python3.10
+      PATH_ORIG: $PATH
       PATH: $PATH:$SNAP/bin:$SNAP/sbin
       # APPORT_DATA_DIR must be set before the `apport` python module is
       # imported.
@@ -152,6 +155,7 @@ parts:
       bin/subiquity-service: usr/bin/subiquity-service
       bin/subiquity-server: usr/bin/subiquity-server
       bin/subiquity-cmd: usr/bin/subiquity-cmd
+      bin/subiquity-umockdev-wrapper: system_scripts/subiquity-umockdev-wrapper
 
     build-attributes:
       - enable-patchelf

subiquity/server/dryrun.py

@@ -80,7 +80,7 @@ class DRConfig:
     ]
 
     # If running ubuntu-drivers on the host, supply a file to
-    # umockdev-wrapper.py
+    # subiquity-umockdev-wrapper
     ubuntu_drivers_run_on_host_umockdev: Optional[
         str
     ] = "examples/umockdev/dell-certified+nvidia.yaml"

subiquity/server/tests/test_ubuntu_drivers.py

@@ -218,7 +218,7 @@ def test_init_with_umockdev(self):
         self.assertEqual(
             ubuntu_drivers.list_oem_cmd,
             [
-                "scripts/umockdev-wrapper.py",
+                "system_scripts/subiquity-umockdev-wrapper",
                 "--config",
                 "/xps.yaml",
                 "--",
@@ -229,7 +229,7 @@ def test_init_with_umockdev(self):
         self.assertEqual(
             ubuntu_drivers.list_drivers_cmd[0:6],
             [
-                "scripts/umockdev-wrapper.py",
+                "system_scripts/subiquity-umockdev-wrapper",
                 "--config",
                 "/xps.yaml",
                 "--",
@@ -240,7 +240,7 @@ def test_init_with_umockdev(self):
         self.assertEqual(
             ubuntu_drivers.install_drivers_cmd[0:6],
             [
-                "scripts/umockdev-wrapper.py",
+                "system_scripts/subiquity-umockdev-wrapper",
                 "--config",
                 "/xps.yaml",
                 "--",

subiquity/server/ubuntu_drivers.py

@@ -16,12 +16,17 @@
 """ Module that defines helpers to use the ubuntu-drivers command. """
 
 import logging
+import os
+import re
 import subprocess
 from abc import ABC, abstractmethod
 from typing import List, Type
 
+import yaml
+
 from subiquity.server.curtin import run_curtin_command
-from subiquitycore.utils import arun_command
+from subiquitycore.file_util import copy_file_if_exists, write_file
+from subiquitycore.utils import arun_command, system_scripts_env
 
 log = logging.getLogger("subiquity.server.ubuntu_drivers")
 
@@ -168,6 +173,146 @@ async def list_oem(self, root_dir: str, context) -> List[str]:
         return self._oem_metapackages_from_output(result.stdout.decode("utf-8"))
 
 
+class UbuntuDriversFakePCIDevicesInterface(UbuntuDriversInterface):
+    """An implementation of ubuntu-drivers that wraps the calls with
+    the umockdev wrapper script.
+
+
+    Requires an online install to download umockdev packages or
+    a modified ISOs with the packages added to the pool.
+    """
+
+    def __init__(self, app, gpgpu: bool) -> None:
+        super().__init__(app, gpgpu)
+
+        # PCI devices can be passed on the kernel command line as a comma
+        # separated list:
+        #  subiquity-fake-pci-devices=pci:v00001234d00001234,pci:v00005678d00005678
+        pcis = app.opts.kernel_cmdline.get("subiquity-fake-pci-devices")
+        devs = [self.modalias_to_config(d) for d in pcis.split(",") if d != ""]
+        self.dev_config = {"devices": devs}
+        log.debug(f"writing umockdev_config: {self.dev_config}")
+
+        # write config to live environment
+        self.dev_config_path = "/tmp/umockdev_config.yaml"
+        write_file(self.dev_config_path, yaml.safe_dump(self.dev_config), mode=0o777)
+
+        prefix: list[str] = [
+            "subiquity-umockdev-wrapper",  # vendored in system_scripts
+            "--config",
+            self.dev_config_path,
+            "--",  # Don't let wrapper consume ubuntu-drivers feature flags
+        ]
+
+        self.sys_env = system_scripts_env()
+
+        self.pre_req_cmd = [
+            "apt-get",
+            "install",
+            "-oDPkg::Lock::Timeout=-1",  # avoid oem vs drivers lock timeout
+            "-y",
+            "umockdev",
+            "gir1.2-umockdev-1.0",
+        ]
+
+        self.list_drivers_cmd = prefix + self.list_drivers_cmd
+        self.list_oem_cmd = prefix + self.list_oem_cmd
+        self.install_drivers_cmd = prefix + self.install_drivers_cmd
+
+    def modalias_to_config(self, modalias: str) -> dict[str, list[dict[str, str]]]:
+        """Generate a device config for umockdev-wrapper given a modalias."""
+        matches = re.compile(
+            r"pci:v(?P<vendor_id>[0-9A-F]{8})d(?P<device_id>[0-9A-F]{8})"
+        ).match(modalias)
+
+        assert matches is not None, f"{modalias} is malformed."
+
+        return {
+            "modalias": modalias,
+            "vendor": f"0x{matches.group('vendor_id')}",
+            "device": f"0x{matches.group('device_id')}",
+        }
+
+    async def ensure_cmd_exists(self, root_dir: str) -> None:
+        # TODO This does not tell us if the "--recommended" option is
+        # available.
+        try:
+            await arun_command(["sh", "-c", "command -v ubuntu-drivers"], check=True)
+        except subprocess.CalledProcessError:
+            raise CommandNotFoundError(
+                f"Command ubuntu-drivers is not available in {root_dir}"
+            )
+        # Install wrapper script prerequisites on live system
+        try:
+            await arun_command(self.pre_req_cmd, check=True)
+        except subprocess.CalledProcessError as err:
+            log.debug(f"ensure_cmd returned with exit code {err.returncode}")
+            log.debug(f"ensure_cmd stdout: {err.stdout}")
+            log.debug(f"ensure_cmd stderr: {err.stderr}")
+            raise Exception("Installing umockdev failed. Quitting early.")
+
+    async def list_drivers(self, root_dir: str, context) -> List[str]:
+        result = await arun_command(self.list_drivers_cmd, env=self.sys_env)
+        return self._drivers_from_output(result.stdout)
+
+    async def list_oem(self, root_dir: str, context) -> List[str]:
+        result = await arun_command(self.list_oem_cmd, env=self.sys_env)
+        return self._oem_metapackages_from_output(result.stdout)
+
+    def _get_wrapper_path(self) -> str | None:
+        # Find subiquity-umockdev-wrapper on live system
+        base_paths: list[str] = self.sys_env["PATH"].split(":")
+        script_path: str | None = None
+
+        log.debug(f"Looking in {base_paths}")
+        for b in base_paths:
+            test_path = f"{b}/subiquity-umockdev-wrapper"
+            log.debug(f"looking for {test_path=}")
+            if os.path.isfile(test_path):
+                script_path = test_path
+                break
+
+        log.debug(f"found {script_path=}")
+
+        return script_path
+
+    async def install_drivers(self, root_dir: str, context) -> None:
+        # Copy config from live system, allowing changes made to the config
+        # after it has been written to persist.
+        copy_file_if_exists(
+            self.dev_config_path,
+            f"{root_dir}/{self.dev_config_path}",
+        )
+
+        # Copy wrapper script to target system
+        # The "find and copy to /target/usr/bin" strategy is to get around
+        # messing with $PATH on curtin in-target commands. When, or if, a more
+        # standard way to run commands inside the target environment comes
+        # along this should be converted to conform.
+        wrapper_script_source: str | None = self._get_wrapper_path()
+        if wrapper_script_source is None:
+            raise Exception("Couldn't find path to subiquity-umockdev-wrapper")
+
+        wrapper_script_dest: str = f"{root_dir}/usr/bin/subiquity-umockdev-wrapper"
+        copy_file_if_exists(wrapper_script_source, wrapper_script_dest)
+        os.chmod(wrapper_script_dest, 0o777)  # Make sure it's executable
+
+        # Install wrapper script pre-reqs on target
+        await run_curtin_command(
+            self.app,
+            context,
+            "in-target",
+            "-t",
+            root_dir,
+            "--",
+            *self.pre_req_cmd,
+            private_mounts=True,
+        )
+
+        # Finally call the wrapped install
+        await super().install_drivers(root_dir, context)
+
+
 class UbuntuDriversHasDriversInterface(UbuntuDriversInterface):
     """A dry-run implementation of ubuntu-drivers that returns a hard-coded
     list of drivers."""
@@ -210,21 +355,21 @@ def __init__(self, app, gpgpu: bool) -> None:
             return
 
         self.list_oem_cmd = [
-            "scripts/umockdev-wrapper.py",
+            "system_scripts/subiquity-umockdev-wrapper",
             "--config",
             app.dr_cfg.ubuntu_drivers_run_on_host_umockdev,
             "--",
         ] + self.list_oem_cmd
 
         self.list_drivers_cmd = [
-            "scripts/umockdev-wrapper.py",
+            "system_scripts/subiquity-umockdev-wrapper",
             "--config",
             app.dr_cfg.ubuntu_drivers_run_on_host_umockdev,
             "--",
         ] + self.list_drivers_cmd
 
         self.install_drivers_cmd = [
-            "scripts/umockdev-wrapper.py",
+            "system_scripts/subiquity-umockdev-wrapper",
             "--config",
             app.dr_cfg.ubuntu_drivers_run_on_host_umockdev,
             "--",
@@ -252,7 +397,7 @@ async def list_oem(self, root_dir: str, context) -> List[str]:
 
 
 def get_ubuntu_drivers_interface(app) -> UbuntuDriversInterface:
-    is_server = app.base_model.source.current.variant == "server"
+    use_gpgpu = app.base_model.source.current.variant == "server"
     cls: Type[UbuntuDriversInterface] = UbuntuDriversClientInterface
     if app.opts.dry_run:
         if "no-drivers" in app.debug_flags:
@@ -262,4 +407,14 @@ def get_ubuntu_drivers_interface(app) -> UbuntuDriversInterface:
         else:
             cls = UbuntuDriversHasDriversInterface
 
-    return cls(app, gpgpu=is_server)
+    if app.opts.kernel_cmdline.get("subiquity-fake-pci-devices"):
+        log.debug("Using umockdev wrapper")
+        cls = UbuntuDriversFakePCIDevicesInterface
+
+    # For quickly testing MOK enrollment we install on server and force no gpgpu
+    # The caveat to this is that it also has to be an online install
+    if "subiquity-server-force-no-gpgpu" in app.opts.kernel_cmdline:
+        log.debug("Forcing no gpgpu drivers. Requires online install on server.")
+        use_gpgpu = False
+
+    return cls(app, gpgpu=use_gpgpu)