nixos-modules: move virtiofsd from host to microvm

lib/runner.nix

@@ -6,6 +6,8 @@
 let
   inherit (pkgs) lib;
 
+  inherit (microvmConfig) virtiofsdScripts;
+
   inherit (import ./. { inherit lib; }) createVolumesScript makeMacvtap;
   inherit (makeMacvtap {
     inherit microvmConfig hypervisorConfig;
@@ -67,6 +69,16 @@ pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${microvmConf
     ln -s ${balloonScriptBin}/bin/microvm-balloon $out/bin/microvm-balloon
   ''}
 
+  ${lib.optionalString (virtiofsdScripts.run != null) ''
+    ln -s ${lib.getExe virtiofsdScripts.run} $out/bin/virtiofsd-run
+  ''}
+  ${lib.optionalString (virtiofsdScripts.reload != null) ''
+    ln -s ${lib.getExe virtiofsdScripts.reload} $out/bin/virtiofsd-reload
+  ''}
+  ${lib.optionalString (virtiofsdScripts.stop != null) ''
+    ln -s ${lib.getExe virtiofsdScripts.stop} $out/bin/virtiofsd-stop
+  ''}
+
   mkdir -p $out/share/microvm
   ln -s ${toplevel} $out/share/microvm/system
 

nixos-modules/host/default.nix

@@ -250,68 +250,37 @@ in
         '';
       };
 
-      "microvm-virtiofsd@" = let
-        inherit (pkgs.python3Packages) supervisor;
-      in rec {
-        description = "VirtioFS daemons for MicroVM '%i'";
-        before = [ "microvm@%i.service" ];
-        after = [ "local-fs.target" ];
-        partOf = [ "microvm@%i.service" ];
-        unitConfig.ConditionPathExists = "${stateDir}/%i/current/share/microvm/virtiofs";
-        restartIfChanged = false;
-        serviceConfig = {
-          ExecReload = "${lib.getExe' supervisor "supervisorctl"} reload";
-          ExecStop = "${lib.getExe' supervisor "supervisorctl"} shutdown";
-          LimitNOFILE = 1048576;
-          NotifyAccess = "all";
-          PrivateTmp = "yes";
-          Restart = "always";
-          RestartSec = "5s";
-          SyslogIdentifier = "microvm-virtiofsd@%i";
-          Type = "notify";
-          WorkingDirectory = "${stateDir}/%i";
-        };
-        script = ''
-          echo "[supervisord]
-          nodaemon=true
-          user=root
-
-          [eventlistener:notify]
-          command=${pkgs.writers.writePython3 "supervisord-event-handler" { } (lib.readFile ./supervisord-event-handler.py)}
-          events=PROCESS_STATE
-          " > /tmp/supervisord.conf
-
-          virtiofsd_count=0
-
-          for d in $PWD/current/share/microvm/virtiofs/*; do
-            SOCKET="$(realpath "$(cat $d/socket)")"
-            SOURCE="$(cat $d/source)"
-            mkdir -p "$SOURCE"
-
-            group_programs+="virtiofsd-$(basename "$d"),"
-            virtiofsd_count=$((virtiofsd_count+1))
-
-            echo "[program:virtiofsd-$(basename "$d")]
-          stderr_syslog=true
-          stdout_syslog=true
-          command=${lib.getExe pkgs.virtiofsd} \
-          --socket-path=$SOCKET \
-          --socket-group=${config.users.users.microvm.group} \
-          --shared-dir=\"$SOURCE\" \
-          --rlimit-nofile ${toString serviceConfig.LimitNOFILE} \
-          --thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \
-          --posix-acl --xattr \
-          ${lib.optionalString (config.microvm.virtiofsd.inodeFileHandles != null)
-          "--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}"
-          } \
-          ${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs}
-          " >> /tmp/supervisord.conf
-          done
+      "microvm-virtiofsd@" =
+        let
+          runFromBootedOrCurrent = name: pkgs.writeShellScript "virtiofsd-${name}" ''
+            if [ -e booted ]; then
+              exec booted/bin/${name}
+            else
+              exec current/bin/${name}
+            fi
+          '';
 
-          echo -n $virtiofsd_count > /tmp/virtiofsd_count
-          exec ${lib.getExe' supervisor "supervisord"} --configuration /tmp/supervisord.conf
-        '';
-      };
+        in {
+          description = "VirtioFS daemons for MicroVM '%i'";
+          before = [ "microvm@%i.service" ];
+          after = [ "local-fs.target" ];
+          partOf = [ "microvm@%i.service" ];
+          unitConfig.ConditionPathExists = "${stateDir}/%i/current/bin/run-virtiofsd";
+          restartIfChanged = false;
+          serviceConfig = {
+            WorkingDirectory = "${stateDir}/%i";
+            ExecStart = "${stateDir}/%i/current/bin/virtiofsd-run";
+            ExecReload = runFromBootedOrCurrent "virtiofsd-reload";
+            ExecStop = runFromBootedOrCurrent "virtiofsd-shutdown";
+            LimitNOFILE = 1048576;
+            NotifyAccess = "all";
+            PrivateTmp = "yes";
+            Restart = "always";
+            RestartSec = "5s";
+            SyslogIdentifier = "microvm-virtiofsd@%i";
+            Type = "notify";
+          };
+        };
 
       "microvm@" = {
         description = "MicroVM '%i'";

nixos-modules/host/options.nix

@@ -169,37 +169,5 @@
         This includes declarative `config.microvm.vms` as well as MicroVMs that are managed through the `microvm` command.
       '';
     };
-
-    virtiofsd.inodeFileHandles = mkOption {
-      type = with types; nullOr (enum [
-        "never" "prefer" "mandatory"
-      ]);
-      default = null;
-      description = ''
-        When to use file handles to reference inodes instead of O_PATH file descriptors
-        (never, prefer, mandatory)
-
-        Allows you to overwrite default behavior in case you hit "too
-        many open files" on eg. ZFS.
-        <https://gitlab.com/virtio-fs/virtiofsd/-/issues/121>
-      '';
-    };
-
-    virtiofsd.threadPoolSize = mkOption {
-      type = with types; oneOf [ str ints.unsigned ];
-      default = "`nproc`";
-      description = ''
-        The amounts of threads virtiofsd should spawn. This option also takes the special
-        string `\`nproc\`` which spawns as many threads as the host has cores.
-      '';
-    };
-
-    virtiofsd.extraArgs = mkOption {
-      type = with types; listOf str;
-      default = [];
-      description = ''
-        Extra command-line switch to pass to virtiofsd.
-      '';
-    };
   };
 }

nixos-modules/microvm/default.nix

@@ -15,6 +15,7 @@ in
     ./asserts.nix
     ./system.nix
     ./mounts.nix
+    ./virtiofsd
     ./graphics.nix
     ./optimization.nix
     ./ssh-deploy.nix

nixos-modules/microvm/options.nix

@@ -461,6 +461,38 @@ in
       '';
     };
 
+    virtiofsd.inodeFileHandles = mkOption {
+      type = with types; nullOr (enum [
+        "never" "prefer" "mandatory"
+      ]);
+      default = null;
+      description = ''
+        When to use file handles to reference inodes instead of O_PATH file descriptors
+        (never, prefer, mandatory)
+
+        Allows you to overwrite default behavior in case you hit "too
+        many open files" on eg. ZFS.
+        <https://gitlab.com/virtio-fs/virtiofsd/-/issues/121>
+      '';
+    };
+
+    virtiofsd.threadPoolSize = mkOption {
+      type = with types; oneOf [ str ints.unsigned ];
+      default = "`nproc`";
+      description = ''
+        The amounts of threads virtiofsd should spawn. This option also takes the special
+        string `\`nproc\`` which spawns as many threads as the host has cores.
+      '';
+    };
+
+    virtiofsd.extraArgs = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = ''
+        Extra command-line switch to pass to virtiofsd.
+      '';
+    };
+
     runner = mkOption {
       description = "Generated Hypervisor runner for this NixOS";
       type = with types; attrsOf package;
@@ -472,6 +504,26 @@ in
       default = config.microvm.runner.${config.microvm.hypervisor};
       defaultText = literalExpression ''"config.microvm.runner.''${config.microvm.hypervisor}"'';
     };
+
+    virtiofsdScripts = {
+      run = mkOption {
+        description = "Generated script to run required virtiofsd instances";
+        type = with types; nullOr package;
+        default = null;
+      };
+      reload = mkOption {
+        description = ''
+          Generated script to reload the supervisor configuration that runs the virtiofsd instances
+        '';
+        type = with types; nullOr package;
+        default = null;
+      };
+      shutdown = mkOption {
+        description = "Generated script to stop the virtiofsd instances";
+        type = with types; nullOr package;
+        default = null;
+      };
+    };
   };
 
   config = lib.mkMerge [ {

nixos-modules/microvm/virtiofsd/default.nix

@@ -0,0 +1,72 @@
+{ config, lib, pkgs, ... }:
+
+let
+  virtiofsShares = builtins.filter ({ proto, ... }:
+    proto == "virtiofs"
+  ) config.microvm.shares;
+
+  requiresVirtiofsd = virtiofsShares != [];
+
+  inherit (pkgs.python3Packages) supervisor;
+  supervisord = lib.getExe' supervisor "supervisord";
+  supervisorctl = lib.getExe' supervisor "supervisorctl";
+
+in
+{
+  microvm.virtiofsdScripts = lib.mkIf requiresVirtiofsd {
+    run =
+      let
+        supervisordConfig = pkgs.writeText "${config.networking.hostName}-virtiofsd-supervisord.conf" ''
+          [supervisord]
+          nodaemon=true
+          #user=root
+          logfile=/dev/stdout
+          logfile_maxbytes=0
+
+          [eventlistener:notify]
+          command=${pkgs.writers.writePython3 "supervisord-event-handler" { } (
+                 pkgs.substituteAll {
+                   src = ./supervisord-event-handler.py;
+                   virtiofsdCount = builtins.length virtiofsShares;
+                 }
+               )}
+          events=PROCESS_STATE
+
+          ${lib.concatMapStrings ({ proto, tag, socket, source, ... }: ''
+            [program:virtiofsd-${tag}]
+            stderr_syslog=true
+            stdout_syslog=true
+            autorestart=true
+            command=${pkgs.writeShellScript "virtiofsd-${tag}" ''
+              if [ $(id -u) = 0 ]; then
+                OPT_RLIMIT="--rlimit-nofile 1048576"
+              else
+                OPT_RLIMIT=""
+              fi
+              exec ${lib.getExe pkgs.virtiofsd} \
+                --socket-path=${lib.escapeShellArg socket} \
+                --socket-group=$(id -gn) \
+                --shared-dir=${lib.escapeShellArg source} \
+                $OPT_RLIMIT \
+                --thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \
+                --posix-acl --xattr \
+                ${lib.optionalString (config.microvm.virtiofsd.inodeFileHandles != null)
+                  "--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}"
+                 } \
+                ${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs}
+              ''}
+          '' ) virtiofsShares}
+        '';
+      in pkgs.writeShellScriptBin "run-virtiofsd" ''
+        exec ${supervisord} --configuration ${supervisordConfig}
+      '';
+
+    reload = pkgs.writeShellScriptBin "reload-virtiofsd" ''
+      exec ${supervisorctl} reload
+    '';
+
+    shutdown = pkgs.writeShellScriptBin "shutdown-virtiofsd" ''
+      exec ${supervisorctl} stop
+    '';
+  };
+}

nixos-modules/host/supervisord-event-handler.py → nixos-modules/microvm/virtiofsd/supervisord-event-handler.py

@@ -15,8 +15,7 @@ def write_stderr(s):
 
 def main():
     count = 0
-    with open('/tmp/virtiofsd_count') as f:
-        expected_count = int(f.read())
+    expected_count = @virtiofsdCount@
 
     while True:
         write_stdout('READY\n')