Merge pull request #29 from desultory/module_compression

Module firmware decompression/xz deps

src/ugrd/base/core.py

@@ -1,5 +1,5 @@
 __author__ = 'desultory'
-__version__ = '3.4.0'
+__version__ = '3.5.0'
 
 from pathlib import Path
 from typing import Union
@@ -91,18 +91,32 @@ def deploy_dependencies(self) -> None:
         self._copy(dependency)
 
 
+def deploy_xz_dependencies(self) -> None:
+    """ Decompresses all xz dependencies into the build directory. """
+    from lzma import decompress
+    for xz_dependency in self['xz_dependencies']:
+        self.logger.debug("[xz] Decompressing: %s" % xz_dependency)
+        out_path = self._get_build_path(str(xz_dependency).replace('.xz', ''))
+        if not out_path.parent.is_dir():
+            self.logger.debug("Creating parent directory: %s" % out_path.parent)
+            self._mkdir(out_path.parent)
+        with out_path.open('wb') as out_file:
+            out_file.write(decompress(xz_dependency.read_bytes()))
+            self.logger.info("[xz] Decompressed '%s' to: %s" % (xz_dependency, out_path))
+
+
 def deploy_gz_dependencies(self) -> None:
     """ Decompresses all gzip dependencies into the build directory. """
     from gzip import decompress
     for gz_dependency in self['gz_dependencies']:
-        self.logger.debug("Decompressing: %s" % gz_dependency)
+        self.logger.debug("[gz] Decompressing: %s" % gz_dependency)
         out_path = self._get_build_path(str(gz_dependency).replace('.gz', ''))
         if not out_path.parent.is_dir():
             self.logger.debug("Creating parent directory: %s" % out_path.parent)
             self._mkdir(out_path.parent)
         with out_path.open('wb') as out_file:
             out_file.write(decompress(gz_dependency.read_bytes()))
-            self.logger.info("Decompressed '%s' to: %s" % (gz_dependency, out_path))
+            self.logger.info("[gz] Decompressed '%s' to: %s" % (gz_dependency, out_path))
 
 
 def deploy_copies(self) -> None:
@@ -239,6 +253,23 @@ def _process_opt_dependencies_multi(self, dependency: Union[Path, str]) -> None:
         self.logger.debug(e)
 
 
+def _process_xz_dependencies_multi(self, dependency: Union[Path, str]) -> None:
+    """
+    Checks that the file is a xz file, and adds it to the xz dependencies list.
+    !! Resolves symlinks implicitly !!
+    """
+    if not isinstance(dependency, Path):
+        dependency = Path(dependency)
+
+    if not dependency.exists():
+        raise FileNotFoundError("XZ dependency does not exist: %s" % dependency)
+
+    if dependency.suffix != '.xz':
+        self.logger.warning("XZ dependency missing xz extension: %s" % dependency)
+
+    self['xz_dependencies'].append(dependency)
+
+
 def _process_gz_dependencies_multi(self, dependency: Union[Path, str]) -> None:
     """
     Checks that the file is a gz file, and adds it to the gz dependencies list.

src/ugrd/base/core.toml

@@ -22,6 +22,7 @@ minor = 1
 		     "_process_binaries_multi",
 		     "_process_dependencies_multi",
 		     "_process_opt_dependencies_multi",
+		     "_process_xz_dependencies_multi",
 		     "_process_gz_dependencies_multi",
 		     "_process_copies_multi",
 		     "_process_symlinks_multi",
@@ -37,7 +38,13 @@ minor = 1
 "ugrd.base.core" = [ "clean_build_dir", "find_libgcc" ]
 
 [imports.build_tasks]
-"ugrd.base.core" = [ "generate_structure", "deploy_dependencies", "deploy_gz_dependencies", "deploy_copies", "deploy_nodes", "deploy_symlinks" ]
+"ugrd.base.core" = [ "generate_structure",
+		     "deploy_dependencies",
+		     "deploy_xz_dependencies",
+		     "deploy_gz_dependencies",
+		     "deploy_copies",
+		     "deploy_nodes",
+		     "deploy_symlinks" ]
 
 [imports.build_final]
 "ugrd.base.core" = [ "check_usr" ]
@@ -53,6 +60,7 @@ _build_log_level = "int"  # The level of logging to use for the build log, set t
 symlinks = "dict"  # Symlinks dict, defines the symlinks to be made in the initramfs
 dependencies = "NoDupFlatList"  # Dependencies, used to define the dependencies of the initramfs
 opt_dependencies = "NoDupFlatList"  # Optional dependencies, which will be included if they are found
+xz_dependencies = "NoDupFlatList"  # XZipped dependencies property, used to define the xzipped dependencies (will be extracted)
 gz_dependencies = "NoDupFlatList"  # GZipped dependencies property, used to define the gzipped dependencies (will be extracted)
 library_paths = "NoDupFlatList"  # library_paths property, used to define the library paths to add to LD_LIBRARY_PATH
 find_libgcc = "bool"  # If true, the initramfs will search for libgcc_s.so.1 and add it to the initramfs

src/ugrd/kmod/kmod.py

@@ -1,5 +1,5 @@
 __author__ = 'desultory'
-__version__ = '2.9.1'
+__version__ = '2.10.0'
 
 from pathlib import Path
 from subprocess import run
@@ -186,13 +186,23 @@ def _add_kmod_firmware(self, kmod: str) -> None:
         return
 
     for firmware in self['_kmod_modinfo'][kmod]['firmware']:
-        firmware_path = Path('/lib/firmware') / firmware
-        if not firmware_path.exists():
+        _add_firmware_dep(self, kmod, firmware)
+
+
+def _add_firmware_dep(self, kmod: str, firmware: str) -> None:
+    """ Adds a kernel module firmware file to the initramfs dependencies. """
+    firmware_path = Path('/lib/firmware') / firmware
+    if not firmware_path.exists():
+        if firmware_path.with_suffix(firmware_path.suffix + '.xz').exists():
+            firmware_path = firmware_path.with_suffix(firmware_path.suffix + '.xz')
+            if self['kmod_decompress_firmware']:  # otherise, just add it like a normal dependency
+                self['xz_dependencies'] = firmware_path
+                return self.logger.debug("[%s] Found xz compressed firmware file: %s" % (kmod, firmware_path))
+        else:
             # Really, this should be a huge error, but with xhci_pci, it wants some renesas firmware that's not in linux-firmware and doesn't seem to matter
-            self.logger.error("[%s] Firmware file does not exist: %s" % (kmod, firmware_path))
-            continue
-        self.logger.debug("[%s] Adding firmware file to dependencies: %s" % (kmod, firmware_path))
-        self['dependencies'] = firmware_path
+            return self.logger.error("[%s] Firmware file does not exist: %s" % (kmod, firmware_path))
+    self.logger.debug("[%s] Adding firmware file to dependencies: %s" % (kmod, firmware_path))
+    self['dependencies'] = firmware_path
 
 
 def _process_kmod_dependencies(self, kmod: str) -> None:

src/ugrd/kmod/kmod.toml

@@ -1,6 +1,7 @@
 binaries = [ "modprobe" ]
 
 kmod_pull_firmware = true
+kmod_decompress_firmware = true
 
 [custom_parameters]
 _kmod_removed = "NoDupFlatList"  # Meant to be used internally, defines kernel modules which have been ignored at runtime
@@ -9,6 +10,7 @@ _kmod_auto = "NoDupFlatList"  # Used internally, defines kernel modules which ha
 kernel_version = "str"  # Kernel version to use for the initramfs
 kmod_ignore = "NoDupFlatList"  # Kernel modules to ignore when loading
 kmod_pull_firmware = "bool"  # Whether or not to pull firmware for kernel modules
+kmod_decompress_firmware = "bool"  # Whether or not to decompress firmware
 kmod_ignore_softdeps = "bool"  # Whether or not softdeps are ignored
 kmod_autodetect_lsmod = "bool"  # Whether or not to automatically pull currently loaded kernel modules
 kmod_autodetect_lspci = "bool"  # Whether or not to automatically pull kernel modules from lspci -k