diff --git a/lib/runner.nix b/lib/runner.nix index 8db5df14..f050affa 100644 --- a/lib/runner.nix +++ b/lib/runner.nix @@ -6,6 +6,8 @@ let inherit (pkgs) lib; + inherit (microvmConfig) virtiofsdScripts; + inherit (import ./. { inherit lib; }) createVolumesScript makeMacvtap; inherit (makeMacvtap { inherit microvmConfig hypervisorConfig; @@ -67,6 +69,16 @@ pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${microvmConf ln -s ${balloonScriptBin}/bin/microvm-balloon $out/bin/microvm-balloon ''} + ${lib.optionalString (virtiofsdScripts.run != null) '' + ln -s ${lib.getExe virtiofsdScripts.run} $out/bin/virtiofsd-run + ''} + ${lib.optionalString (virtiofsdScripts.reload != null) '' + ln -s ${lib.getExe virtiofsdScripts.reload} $out/bin/virtiofsd-reload + ''} + ${lib.optionalString (virtiofsdScripts.stop != null) '' + ln -s ${lib.getExe virtiofsdScripts.stop} $out/bin/virtiofsd-stop + ''} + mkdir -p $out/share/microvm ln -s ${toplevel} $out/share/microvm/system diff --git a/nixos-modules/host/default.nix b/nixos-modules/host/default.nix index cac6a464..a5962e5f 100644 --- a/nixos-modules/host/default.nix +++ b/nixos-modules/host/default.nix @@ -250,68 +250,37 @@ in ''; }; - "microvm-virtiofsd@" = let - inherit (pkgs.python3Packages) supervisor; - in rec { - description = "VirtioFS daemons for MicroVM '%i'"; - before = [ "microvm@%i.service" ]; - after = [ "local-fs.target" ]; - partOf = [ "microvm@%i.service" ]; - unitConfig.ConditionPathExists = "${stateDir}/%i/current/share/microvm/virtiofs"; - restartIfChanged = false; - serviceConfig = { - ExecReload = "${lib.getExe' supervisor "supervisorctl"} reload"; - ExecStop = "${lib.getExe' supervisor "supervisorctl"} shutdown"; - LimitNOFILE = 1048576; - NotifyAccess = "all"; - PrivateTmp = "yes"; - Restart = "always"; - RestartSec = "5s"; - SyslogIdentifier = "microvm-virtiofsd@%i"; - Type = "notify"; - WorkingDirectory = "${stateDir}/%i"; - }; - script = '' - echo "[supervisord] - nodaemon=true - user=root - - [eventlistener:notify] - command=${pkgs.writers.writePython3 "supervisord-event-handler" { } (lib.readFile ./supervisord-event-handler.py)} - events=PROCESS_STATE - " > /tmp/supervisord.conf - - virtiofsd_count=0 - - for d in $PWD/current/share/microvm/virtiofs/*; do - SOCKET="$(realpath "$(cat $d/socket)")" - SOURCE="$(cat $d/source)" - mkdir -p "$SOURCE" - - group_programs+="virtiofsd-$(basename "$d")," - virtiofsd_count=$((virtiofsd_count+1)) - - echo "[program:virtiofsd-$(basename "$d")] - stderr_syslog=true - stdout_syslog=true - command=${lib.getExe pkgs.virtiofsd} \ - --socket-path=$SOCKET \ - --socket-group=${config.users.users.microvm.group} \ - --shared-dir=\"$SOURCE\" \ - --rlimit-nofile ${toString serviceConfig.LimitNOFILE} \ - --thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \ - --posix-acl --xattr \ - ${lib.optionalString (config.microvm.virtiofsd.inodeFileHandles != null) - "--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}" - } \ - ${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs} - " >> /tmp/supervisord.conf - done + "microvm-virtiofsd@" = + let + runFromBootedOrCurrent = name: pkgs.writeShellScript "virtiofsd-${name}" '' + if [ -e booted ]; then + exec booted/bin/${name} + else + exec current/bin/${name} + fi + ''; - echo -n $virtiofsd_count > /tmp/virtiofsd_count - exec ${lib.getExe' supervisor "supervisord"} --configuration /tmp/supervisord.conf - ''; - }; + in { + description = "VirtioFS daemons for MicroVM '%i'"; + before = [ "microvm@%i.service" ]; + after = [ "local-fs.target" ]; + partOf = [ "microvm@%i.service" ]; + unitConfig.ConditionPathExists = "${stateDir}/%i/current/bin/run-virtiofsd"; + restartIfChanged = false; + serviceConfig = { + WorkingDirectory = "${stateDir}/%i"; + ExecStart = "${stateDir}/%i/current/bin/virtiofsd-run"; + ExecReload = runFromBootedOrCurrent "virtiofsd-reload"; + ExecStop = runFromBootedOrCurrent "virtiofsd-shutdown"; + LimitNOFILE = 1048576; + NotifyAccess = "all"; + PrivateTmp = "yes"; + Restart = "always"; + RestartSec = "5s"; + SyslogIdentifier = "microvm-virtiofsd@%i"; + Type = "notify"; + }; + }; "microvm@" = { description = "MicroVM '%i'"; diff --git a/nixos-modules/host/options.nix b/nixos-modules/host/options.nix index e172ba2d..9775a83b 100644 --- a/nixos-modules/host/options.nix +++ b/nixos-modules/host/options.nix @@ -169,37 +169,5 @@ This includes declarative `config.microvm.vms` as well as MicroVMs that are managed through the `microvm` command. ''; }; - - virtiofsd.inodeFileHandles = mkOption { - type = with types; nullOr (enum [ - "never" "prefer" "mandatory" - ]); - default = null; - description = '' - When to use file handles to reference inodes instead of O_PATH file descriptors - (never, prefer, mandatory) - - Allows you to overwrite default behavior in case you hit "too - many open files" on eg. ZFS. - - ''; - }; - - virtiofsd.threadPoolSize = mkOption { - type = with types; oneOf [ str ints.unsigned ]; - default = "`nproc`"; - description = '' - The amounts of threads virtiofsd should spawn. This option also takes the special - string `\`nproc\`` which spawns as many threads as the host has cores. - ''; - }; - - virtiofsd.extraArgs = mkOption { - type = with types; listOf str; - default = []; - description = '' - Extra command-line switch to pass to virtiofsd. - ''; - }; }; } diff --git a/nixos-modules/microvm/default.nix b/nixos-modules/microvm/default.nix index 8a5eafcf..2feb1ba4 100644 --- a/nixos-modules/microvm/default.nix +++ b/nixos-modules/microvm/default.nix @@ -15,6 +15,7 @@ in ./asserts.nix ./system.nix ./mounts.nix + ./virtiofsd ./graphics.nix ./optimization.nix ./ssh-deploy.nix diff --git a/nixos-modules/microvm/options.nix b/nixos-modules/microvm/options.nix index 7c631a8e..341c3351 100644 --- a/nixos-modules/microvm/options.nix +++ b/nixos-modules/microvm/options.nix @@ -461,6 +461,38 @@ in ''; }; + virtiofsd.inodeFileHandles = mkOption { + type = with types; nullOr (enum [ + "never" "prefer" "mandatory" + ]); + default = null; + description = '' + When to use file handles to reference inodes instead of O_PATH file descriptors + (never, prefer, mandatory) + + Allows you to overwrite default behavior in case you hit "too + many open files" on eg. ZFS. + + ''; + }; + + virtiofsd.threadPoolSize = mkOption { + type = with types; oneOf [ str ints.unsigned ]; + default = "`nproc`"; + description = '' + The amounts of threads virtiofsd should spawn. This option also takes the special + string `\`nproc\`` which spawns as many threads as the host has cores. + ''; + }; + + virtiofsd.extraArgs = mkOption { + type = with types; listOf str; + default = []; + description = '' + Extra command-line switch to pass to virtiofsd. + ''; + }; + runner = mkOption { description = "Generated Hypervisor runner for this NixOS"; type = with types; attrsOf package; @@ -472,6 +504,26 @@ in default = config.microvm.runner.${config.microvm.hypervisor}; defaultText = literalExpression ''"config.microvm.runner.''${config.microvm.hypervisor}"''; }; + + virtiofsdScripts = { + run = mkOption { + description = "Generated script to run required virtiofsd instances"; + type = with types; nullOr package; + default = null; + }; + reload = mkOption { + description = '' + Generated script to reload the supervisor configuration that runs the virtiofsd instances + ''; + type = with types; nullOr package; + default = null; + }; + shutdown = mkOption { + description = "Generated script to stop the virtiofsd instances"; + type = with types; nullOr package; + default = null; + }; + }; }; config = lib.mkMerge [ { diff --git a/nixos-modules/microvm/virtiofsd/default.nix b/nixos-modules/microvm/virtiofsd/default.nix new file mode 100644 index 00000000..c4fe8b21 --- /dev/null +++ b/nixos-modules/microvm/virtiofsd/default.nix @@ -0,0 +1,72 @@ +{ config, lib, pkgs, ... }: + +let + virtiofsShares = builtins.filter ({ proto, ... }: + proto == "virtiofs" + ) config.microvm.shares; + + requiresVirtiofsd = virtiofsShares != []; + + inherit (pkgs.python3Packages) supervisor; + supervisord = lib.getExe' supervisor "supervisord"; + supervisorctl = lib.getExe' supervisor "supervisorctl"; + +in +{ + microvm.virtiofsdScripts = lib.mkIf requiresVirtiofsd { + run = + let + supervisordConfig = pkgs.writeText "${config.networking.hostName}-virtiofsd-supervisord.conf" '' + [supervisord] + nodaemon=true + #user=root + logfile=/dev/stdout + logfile_maxbytes=0 + + [eventlistener:notify] + command=${pkgs.writers.writePython3 "supervisord-event-handler" { } ( + pkgs.substituteAll { + src = ./supervisord-event-handler.py; + virtiofsdCount = builtins.length virtiofsShares; + } + )} + events=PROCESS_STATE + + ${lib.concatMapStrings ({ proto, tag, socket, source, ... }: '' + [program:virtiofsd-${tag}] + stderr_syslog=true + stdout_syslog=true + autorestart=true + command=${pkgs.writeShellScript "virtiofsd-${tag}" '' + if [ $(id -u) = 0 ]; then + OPT_RLIMIT="--rlimit-nofile 1048576" + else + OPT_RLIMIT="" + fi + exec ${lib.getExe pkgs.virtiofsd} \ + --socket-path=${lib.escapeShellArg socket} \ + --socket-group=$(id -gn) \ + --shared-dir=${lib.escapeShellArg source} \ + $OPT_RLIMIT \ + --thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \ + --posix-acl --xattr \ + ${lib.optionalString (config.microvm.virtiofsd.inodeFileHandles != null) + "--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}" + } \ + ${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs} + ''} + '' ) virtiofsShares} + ''; + in pkgs.writeShellScriptBin "run-virtiofsd" '' + exec ${supervisord} --configuration ${supervisordConfig} + ''; + + reload = pkgs.writeShellScriptBin "reload-virtiofsd" '' + exec ${supervisorctl} reload + ''; + + shutdown = pkgs.writeShellScriptBin "shutdown-virtiofsd" '' + exec ${supervisorctl} stop + ''; + }; +} diff --git a/nixos-modules/host/supervisord-event-handler.py b/nixos-modules/microvm/virtiofsd/supervisord-event-handler.py similarity index 93% rename from nixos-modules/host/supervisord-event-handler.py rename to nixos-modules/microvm/virtiofsd/supervisord-event-handler.py index 76afdf61..ce6f0816 100644 --- a/nixos-modules/host/supervisord-event-handler.py +++ b/nixos-modules/microvm/virtiofsd/supervisord-event-handler.py @@ -15,8 +15,7 @@ def write_stderr(s): def main(): count = 0 - with open('/tmp/virtiofsd_count') as f: - expected_count = int(f.read()) + expected_count = @virtiofsdCount@ while True: write_stdout('READY\n')