nixos/systemd-boot: Add support for an XBOOTLDR partition

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -158,6 +158,10 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
 
 - Cinnamon has been updated to 6.0. Please beware that the [Wayland session](https://blog.linuxmint.com/?p=4591) is still experimental in this release.
 
+- New `boot.loader.systemd-boot.xbootldrMountPoint` allows setting up a separate [XBOOTLDR partition](https://uapi-group.org/specifications/specs/boot_loader_specification/) to store boot files. Useful on systems with a small EFI System partition that cannot be easily repartitioned.
+
+- `boot.loader.systemd-boot` will now verify that `efiSysMountPoint` (and `xbootldrMountPoint` if configured) are mounted partitions.
+
 - `services.postgresql.extraPlugins` changed its type from just a list of packages to also a function that returns such a list.
   For example a config line like ``services.postgresql.extraPlugins = with pkgs.postgresql_11.pkgs; [ postgis ];`` is recommended to be changed to ``services.postgresql.extraPlugins = ps: with ps; [ postgis ];``;
 

nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py

@@ -17,6 +17,8 @@
 
 # These values will be replaced with actual values during the package build
 EFI_SYS_MOUNT_POINT = "@efiSysMountPoint@"
+BOOT_MOUNT_POINT = "@bootMountPoint@"
+LOADER_CONF = f"{EFI_SYS_MOUNT_POINT}/loader/loader.conf"  # Always stored on the ESP
 TIMEOUT = "@timeout@"
 EDITOR = bool("@editor@")
 CONSOLE_MODE = "@consoleMode@"
@@ -28,6 +30,7 @@
 CAN_TOUCH_EFI_VARIABLES = "@canTouchEfiVariables@"
 GRACEFUL = "@graceful@"
 COPY_EXTRA_FILES = "@copyExtraFiles@"
+CHECK_MOUNTPOINTS = "@checkMountpoints@"
 
 @dataclass
 class BootSpec:
@@ -87,7 +90,7 @@ def generation_conf_filename(profile: str | None, generation: int, specialisatio
 
 
 def write_loader_conf(profile: str | None, generation: int, specialisation: str | None) -> None:
-    with open(f"{EFI_SYS_MOUNT_POINT}/loader/loader.conf.tmp", 'w') as f:
+    with open(f"{LOADER_CONF}.tmp", 'w') as f:
         if TIMEOUT != "":
             f.write(f"timeout {TIMEOUT}\n")
         f.write("default %s\n" % generation_conf_filename(profile, generation, specialisation))
@@ -96,7 +99,7 @@ def write_loader_conf(profile: str | None, generation: int, specialisation: str
         f.write(f"console-mode {CONSOLE_MODE}\n")
         f.flush()
         os.fsync(f.fileno())
-    os.rename(f"{EFI_SYS_MOUNT_POINT}/loader/loader.conf.tmp", f"{EFI_SYS_MOUNT_POINT}/loader/loader.conf")
+    os.rename(f"{LOADER_CONF}.tmp", LOADER_CONF)
 
 
 def get_bootspec(profile: str | None, generation: int) -> BootSpec:
@@ -128,7 +131,7 @@ def copy_from_file(file: str, dry_run: bool = False) -> str:
     store_dir = os.path.basename(os.path.dirname(store_file_path))
     efi_file_path = "/efi/nixos/%s-%s.efi" % (store_dir, suffix)
     if not dry_run:
-        copy_if_not_exists(store_file_path, f"{EFI_SYS_MOUNT_POINT}%s" % (efi_file_path))
+        copy_if_not_exists(store_file_path, f"{BOOT_MOUNT_POINT}%s" % (efi_file_path))
     return efi_file_path
 
 def write_entry(profile: str | None, generation: int, specialisation: str | None,
@@ -145,7 +148,7 @@ def write_entry(profile: str | None, generation: int, specialisation: str | None
 
     try:
         if bootspec.initrdSecrets is not None:
-            subprocess.check_call([bootspec.initrdSecrets, f"{EFI_SYS_MOUNT_POINT}%s" % (initrd)])
+            subprocess.check_call([bootspec.initrdSecrets, f"{BOOT_MOUNT_POINT}%s" % (initrd)])
     except subprocess.CalledProcessError:
         if current:
             print("failed to create initrd secrets!", file=sys.stderr)
@@ -155,7 +158,7 @@ def write_entry(profile: str | None, generation: int, specialisation: str | None
                   f'for "{title} - Configuration {generation}", an older generation', file=sys.stderr)
             print("note: this is normal after having removed "
                   "or renamed a file in `boot.initrd.secrets`", file=sys.stderr)
-    entry_file = f"{EFI_SYS_MOUNT_POINT}/loader/entries/%s" % (
+    entry_file = f"{BOOT_MOUNT_POINT}/loader/entries/%s" % (
         generation_conf_filename(profile, generation, specialisation))
     tmp_path = "%s.tmp" % (entry_file)
     kernel_params = "init=%s " % bootspec.init
@@ -202,14 +205,14 @@ def get_generations(profile: str | None = None) -> list[SystemIdentifier]:
 
 
 def remove_old_entries(gens: list[SystemIdentifier]) -> None:
-    rex_profile = re.compile(r"^" + re.escape(EFI_SYS_MOUNT_POINT) + "/loader/entries/nixos-(.*)-generation-.*\.conf$")
-    rex_generation = re.compile(r"^" + re.escape(EFI_SYS_MOUNT_POINT) + "/loader/entries/nixos.*-generation-([0-9]+)(-specialisation-.*)?\.conf$")
+    rex_profile = re.compile(r"^" + re.escape(BOOT_MOUNT_POINT) + "/loader/entries/nixos-(.*)-generation-.*\.conf$")
+    rex_generation = re.compile(r"^" + re.escape(BOOT_MOUNT_POINT) + "/loader/entries/nixos.*-generation-([0-9]+)(-specialisation-.*)?\.conf$")
     known_paths = []
     for gen in gens:
         bootspec = get_bootspec(gen.profile, gen.generation)
         known_paths.append(copy_from_file(bootspec.kernel, True))
         known_paths.append(copy_from_file(bootspec.initrd, True))
-    for path in glob.iglob(f"{EFI_SYS_MOUNT_POINT}/loader/entries/nixos*-generation-[1-9]*.conf"):
+    for path in glob.iglob(f"{BOOT_MOUNT_POINT}/loader/entries/nixos*-generation-[1-9]*.conf"):
         if rex_profile.match(path):
             prof = rex_profile.sub(r"\1", path)
         else:
@@ -220,11 +223,18 @@ def remove_old_entries(gens: list[SystemIdentifier]) -> None:
             continue
         if not (prof, gen_number, None) in gens:
             os.unlink(path)
-    for path in glob.iglob(f"{EFI_SYS_MOUNT_POINT}/efi/nixos/*"):
+    for path in glob.iglob(f"{BOOT_MOUNT_POINT}/efi/nixos/*"):
         if not path in known_paths and not os.path.isdir(path):
             os.unlink(path)
 
 
+def cleanup_esp() -> None:
+    for path in glob.iglob(f"{EFI_SYS_MOUNT_POINT}/loader/entries/nixos*"):
+        os.unlink(path)
+    if os.path.isdir(f"{EFI_SYS_MOUNT_POINT}/efi/nixos"):
+        shutil.rmtree(f"{EFI_SYS_MOUNT_POINT}/efi/nixos")
+
+
 def get_profiles() -> list[str]:
     if os.path.isdir("/nix/var/nix/profiles/system-profiles/"):
         return [x
@@ -255,6 +265,9 @@ def install_bootloader(args: argparse.Namespace) -> None:
     # flags to pass to bootctl install/update
     bootctl_flags = []
 
+    if BOOT_MOUNT_POINT != EFI_SYS_MOUNT_POINT:
+        bootctl_flags.append(f"--boot-path={BOOT_MOUNT_POINT}")
+
     if CAN_TOUCH_EFI_VARIABLES != "1":
         bootctl_flags.append("--no-variables")
 
@@ -263,8 +276,8 @@ def install_bootloader(args: argparse.Namespace) -> None:
 
     if os.getenv("NIXOS_INSTALL_BOOTLOADER") == "1":
         # bootctl uses fopen() with modes "wxe" and fails if the file exists.
-        if os.path.exists(f"{EFI_SYS_MOUNT_POINT}/loader/loader.conf"):
-            os.unlink(f"{EFI_SYS_MOUNT_POINT}/loader/loader.conf")
+        if os.path.exists(LOADER_CONF):
+            os.unlink(LOADER_CONF)
 
         subprocess.check_call([f"{SYSTEMD}/bin/bootctl", f"--esp-path={EFI_SYS_MOUNT_POINT}"] + bootctl_flags + ["install"])
     else:
@@ -291,13 +304,15 @@ def install_bootloader(args: argparse.Namespace) -> None:
             print("updating systemd-boot from %s to %s" % (installed_version, available_version))
             subprocess.check_call([f"{SYSTEMD}/bin/bootctl", f"--esp-path={EFI_SYS_MOUNT_POINT}"] + bootctl_flags + ["update"])
 
-    os.makedirs(f"{EFI_SYS_MOUNT_POINT}/efi/nixos", exist_ok=True)
-    os.makedirs(f"{EFI_SYS_MOUNT_POINT}/loader/entries", exist_ok=True)
+    os.makedirs(f"{BOOT_MOUNT_POINT}/efi/nixos", exist_ok=True)
+    os.makedirs(f"{BOOT_MOUNT_POINT}/loader/entries", exist_ok=True)
 
     gens = get_generations()
     for profile in get_profiles():
         gens += get_generations(profile)
+
     remove_old_entries(gens)
+
     for gen in gens:
         try:
             bootspec = get_bootspec(gen.profile, gen.generation)
@@ -315,9 +330,15 @@ def install_bootloader(args: argparse.Namespace) -> None:
             else:
                 raise e
 
-    for root, _, files in os.walk(f"{EFI_SYS_MOUNT_POINT}/efi/nixos/.extra-files", topdown=False):
-        relative_root = root.removeprefix(f"{EFI_SYS_MOUNT_POINT}/efi/nixos/.extra-files").removeprefix("/")
-        actual_root = os.path.join(f"{EFI_SYS_MOUNT_POINT}", relative_root)
+    if BOOT_MOUNT_POINT != EFI_SYS_MOUNT_POINT:
+        # Cleanup any entries in ESP if xbootldrMountPoint is set.
+        # If the user later unsets xbootldrMountPoint, entries in XBOOTLDR will not be cleaned up
+        # automatically, as we don't have information about the mount point anymore.
+        cleanup_esp()
+
+    for root, _, files in os.walk(f"{BOOT_MOUNT_POINT}/efi/nixos/.extra-files", topdown=False):
+        relative_root = root.removeprefix(f"{BOOT_MOUNT_POINT}/efi/nixos/.extra-files").removeprefix("/")
+        actual_root = os.path.join(f"{BOOT_MOUNT_POINT}", relative_root)
 
         for file in files:
             actual_file = os.path.join(actual_root, file)
@@ -330,7 +351,7 @@ def install_bootloader(args: argparse.Namespace) -> None:
             os.rmdir(actual_root)
         os.rmdir(root)
 
-    os.makedirs(f"{EFI_SYS_MOUNT_POINT}/efi/nixos/.extra-files", exist_ok=True)
+    os.makedirs(f"{BOOT_MOUNT_POINT}/efi/nixos/.extra-files", exist_ok=True)
 
     subprocess.check_call(COPY_EXTRA_FILES)
 
@@ -340,16 +361,23 @@ def main() -> None:
     parser.add_argument('default_config', metavar='DEFAULT-CONFIG', help=f"The default {DISTRO_NAME} config to boot")
     args = parser.parse_args()
 
+    subprocess.check_call(CHECK_MOUNTPOINTS)
+
     try:
         install_bootloader(args)
     finally:
         # Since fat32 provides little recovery facilities after a crash,
         # it can leave the system in an unbootable state, when a crash/outage
         # happens shortly after an update. To decrease the likelihood of this
         # event sync the efi filesystem after each update.
-        rc = libc.syncfs(os.open(f"{EFI_SYS_MOUNT_POINT}", os.O_RDONLY))
+        rc = libc.syncfs(os.open(f"{BOOT_MOUNT_POINT}", os.O_RDONLY))
         if rc != 0:
-            print(f"could not sync {EFI_SYS_MOUNT_POINT}: {os.strerror(rc)}", file=sys.stderr)
+            print(f"could not sync {BOOT_MOUNT_POINT}: {os.strerror(rc)}", file=sys.stderr)
+
+        if BOOT_MOUNT_POINT != EFI_SYS_MOUNT_POINT:
+            rc = libc.syncfs(os.open(EFI_SYS_MOUNT_POINT, os.O_RDONLY))
+            if rc != 0:
+                print(f"could not sync {EFI_SYS_MOUNT_POINT}: {os.strerror(rc)}", file=sys.stderr)
 
 
 if __name__ == '__main__':

nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix

@@ -7,7 +7,7 @@ let
 
   efi = config.boot.loader.efi;
 
-  systemdBootBuilder = pkgs.substituteAll {
+  systemdBootBuilder = pkgs.substituteAll rec {
     src = ./systemd-boot-builder.py;
 
     isExecutable = true;
@@ -30,23 +30,38 @@ let
 
     inherit (efi) efiSysMountPoint canTouchEfiVariables;
 
+    bootMountPoint = if cfg.xbootldrMountPoint != null
+      then cfg.xbootldrMountPoint
+      else efi.efiSysMountPoint;
+
     inherit (config.system.nixos) distroName;
 
     memtest86 = optionalString cfg.memtest86.enable pkgs.memtest86plus;
 
     netbootxyz = optionalString cfg.netbootxyz.enable pkgs.netbootxyz-efi;
 
+    checkMountpoints = pkgs.writeShellScript "check-mountpoints" ''
+      fail() {
+        echo "$1 = '$2' is not a mounted partition. Is the path configured correctly?" >&2
+        exit 1
+      }
+      ${pkgs.util-linuxMinimal}/bin/findmnt ${efiSysMountPoint} > /dev/null || fail efiSysMountPoint ${efiSysMountPoint}
+      ${lib.optionalString
+        (cfg.xbootldrMountPoint != null)
+        "${pkgs.util-linuxMinimal}/bin/findmnt ${cfg.xbootldrMountPoint} > /dev/null || fail xbootldrMountPoint ${cfg.xbootldrMountPoint}"}
+    '';
+
     copyExtraFiles = pkgs.writeShellScript "copy-extra-files" ''
       empty_file=$(${pkgs.coreutils}/bin/mktemp)
 
       ${concatStrings (mapAttrsToList (n: v: ''
-        ${pkgs.coreutils}/bin/install -Dp "${v}" "${efi.efiSysMountPoint}/"${escapeShellArg n}
-        ${pkgs.coreutils}/bin/install -D $empty_file "${efi.efiSysMountPoint}/efi/nixos/.extra-files/"${escapeShellArg n}
+        ${pkgs.coreutils}/bin/install -Dp "${v}" "${bootMountPoint}/"${escapeShellArg n}
+        ${pkgs.coreutils}/bin/install -D $empty_file "${bootMountPoint}/EFI/nixos/.extra-files/"${escapeShellArg n}
       '') cfg.extraFiles)}
 
       ${concatStrings (mapAttrsToList (n: v: ''
-        ${pkgs.coreutils}/bin/install -Dp "${pkgs.writeText n v}" "${efi.efiSysMountPoint}/loader/entries/"${escapeShellArg n}
-        ${pkgs.coreutils}/bin/install -D $empty_file "${efi.efiSysMountPoint}/efi/nixos/.extra-files/loader/entries/"${escapeShellArg n}
+        ${pkgs.coreutils}/bin/install -Dp "${pkgs.writeText n v}" "${bootMountPoint}/loader/entries/"${escapeShellArg n}
+        ${pkgs.coreutils}/bin/install -D $empty_file "${bootMountPoint}/EFI/nixos/.extra-files/loader/entries/"${escapeShellArg n}
       '') cfg.extraEntries)}
     '';
   };
@@ -102,6 +117,18 @@ in {
       '';
     };
 
+    xbootldrMountPoint = mkOption {
+      default = null;
+      type = types.nullOr types.str;
+      description = lib.mdDoc ''
+        Where the XBOOTLDR partition is mounted.
+
+        If set, this partition will be used as $BOOT to store boot loader entries and extra files
+        instead of the EFI partition. As per the bootloader specification, it is recommended that
+        the EFI and XBOOTLDR partitions be mounted at `/efi` and `/boot`, respectively.
+      '';
+    };
+
     configurationLimit = mkOption {
       default = null;
       example = 120;
@@ -111,7 +138,7 @@ in {
         Useful to prevent boot partition running out of disk space.
 
         `null` means no limit i.e. all generations
-        that were not garbage collected yet.
+        that have not been garbage collected yet.
       '';
     };
 
@@ -203,7 +230,7 @@ in {
       '';
       description = lib.mdDoc ''
         Any additional entries you want added to the `systemd-boot` menu.
-        These entries will be copied to {file}`/boot/loader/entries`.
+        These entries will be copied to {file}`$BOOT/loader/entries`.
         Each attribute name denotes the destination file name,
         and the corresponding attribute value is the contents of the entry.
 
@@ -220,9 +247,9 @@ in {
         { "efi/memtest86/memtest.efi" = "''${pkgs.memtest86plus}/memtest.efi"; }
       '';
       description = lib.mdDoc ''
-        A set of files to be copied to {file}`/boot`.
+        A set of files to be copied to {file}`$BOOT`.
         Each attribute name denotes the destination file name in
-        {file}`/boot`, while the corresponding
+        {file}`$BOOT`, while the corresponding
         attribute value specifies the source file.
       '';
     };
@@ -246,6 +273,18 @@ in {
 
   config = mkIf cfg.enable {
     assertions = [
+      {
+        assertion = (hasPrefix "/" efi.efiSysMountPoint);
+        message = "The ESP mount point '${efi.efiSysMountPoint}' must be an absolute path";
+      }
+      {
+        assertion = cfg.xbootldrMountPoint == null || (hasPrefix "/" cfg.xbootldrMountPoint);
+        message = "The XBOOTLDR mount point '${cfg.xbootldrMountPoint}' must be an absolute path";
+      }
+      {
+        assertion = cfg.xbootldrMountPoint != efi.efiSysMountPoint;
+        message = "The XBOOTLDR mount point '${cfg.xbootldrMountPoint}' cannot be the same as the ESP mount point '${efi.efiSysMountPoint}'";
+      }
       {
         assertion = (config.boot.kernelPackages.kernel.features or { efiBootStub = true; }) ? efiBootStub;
         message = "This kernel does not support the EFI boot stub";

nixos/tests/systemd-boot.nix

@@ -14,6 +14,14 @@ let
     boot.loader.efi.canTouchEfiVariables = true;
     environment.systemPackages = [ pkgs.efibootmgr ];
   };
+
+  common_xbootldr = {
+    imports = [ common ];
+
+    virtualisation.useXbootldr = true;
+    boot.loader.efi.efiSysMountPoint = "/efi";
+    boot.loader.systemd-boot.xbootldrMountPoint = "/boot";
+  };
 in
 {
   basic = makeTest {
@@ -39,6 +47,30 @@ in
     '';
   };
 
+  basic_xbootldr = makeTest {
+    name = "systemd-boot-xbootldr";
+    meta.maintainers = with pkgs.lib.maintainers; [ sdht0 ];
+
+    nodes.machine = common_xbootldr;
+
+    testScript = ''
+      machine.start()
+      machine.wait_for_unit("multi-user.target")
+
+      machine.succeed("test -e /efi/EFI/systemd/systemd-bootx64.efi")
+      machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf")
+
+      # Ensure we actually booted using systemd-boot
+      # Magic number is the vendor UUID used by systemd-boot.
+      machine.succeed(
+          "test -e /sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
+      )
+
+      # "bootctl install" should have created an EFI entry
+      machine.succeed('efibootmgr | grep "Linux Boot Manager"')
+    '';
+  };
+
   # Check that specialisations create corresponding boot entries.
   specialisation = makeTest {
     name = "systemd-boot-specialisation";
@@ -158,6 +190,23 @@ in
     '';
   };
 
+  entryFilename_xbootldr = makeTest {
+    name = "systemd-boot-entry-filename";
+    meta.maintainers = with pkgs.lib.maintainers; [ sdht0 ];
+
+    nodes.machine = { pkgs, lib, ... }: {
+      imports = [ common_xbootldr ];
+      boot.loader.systemd-boot.memtest86.enable = true;
+      boot.loader.systemd-boot.memtest86.entryFilename = "apple.conf";
+    };
+
+    testScript = ''
+      machine.fail("test -e /boot/loader/entries/memtest86.conf")
+      machine.succeed("test -e /boot/loader/entries/apple.conf")
+      machine.succeed("test -e /boot/efi/memtest86/memtest.efi")
+    '';
+  };
+
   extraEntries = makeTest {
     name = "systemd-boot-extra-entries";
     meta.maintainers = with pkgs.lib.maintainers; [ Enzime julienmalka ];