libkrun: (gpu/fs) introduce variants set win size

Introduce krun_add_virtiofs2 and krun_set_gpu_options2 as variants of
krun_add_virtiofs and krun_set_gpu_options respectively, to allow users
to specify the DAX/host window size for the device.

We're adding variants of existing functions to avoid breaking existing
consumers of the library. This isn't great, but we'll do a huge clean up
of the API in 2.0.

Signed-off-by: Sergio Lopez <[email protected]>

include/libkrun.h

@@ -121,6 +121,24 @@ int32_t krun_add_virtiofs(uint32_t ctx_id,
                           const char *c_tag,
                           const char *c_path);
 
+/**
+ * Adds an independent virtio-fs device pointing to a host's directory with a tag. This
+ * variant allows specifying the size of the DAX window.
+ *
+ * Arguments:
+ *  "ctx_id"         - the configuration context ID.
+ *  "c_tag"          - tag to identify the filesystem in the guest.
+ *  "c_path"         - full path to the directory in the host to be exposed to the guest.
+ *  "window_size"    - size of the DAX window in bytes.
+ *
+ * Returns:
+ *  Zero on success or a negative error number on failure.
+ */
+int32_t krun_add_virtiofs2(uint32_t ctx_id,
+                           const char *c_tag,
+                           const char *c_path,
+                           uint64_t window_size);
+
 /**
  * Configures the networking to use passt.
  * Call to this function disables TSI backend to use passt instead.
@@ -154,7 +172,7 @@ int32_t krun_set_passt_fd(uint32_t ctx_id, int fd);
  * Returns:
  *  Zero on success or a negative error number on failure.
  */
-int32_t krun_set_gvproxy_path(uint32_t ctx_id, char* c_path);
+int32_t krun_set_gvproxy_path(uint32_t ctx_id, char *c_path);
 
 /**
  * Sets the MAC address for the virtio-net device when using the passt backend.
@@ -219,6 +237,22 @@ int32_t krun_set_port_map(uint32_t ctx_id, const char *const port_map[]);
  */
 int32_t krun_set_gpu_options(uint32_t ctx_id, uint32_t virgl_flags);
 
+/**
+ * Enables and configures a virtio-gpu device. This variant allows specifying
+ * the size of the host window (acting as vRAM in the guest).
+ *
+ * Arguments:
+ *  "ctx_id"      - the configuration context ID.
+ *  "virgl_flags" - flags to pass to virglrenderer.
+ *  "window_size" - size of the host window in bytes.
+ *
+ * Returns:
+ *  Zero on success or a negative error number on failure.
+ */
+int32_t krun_set_gpu_options2(uint32_t ctx_id,
+                              uint32_t virgl_flags,
+                              uint64_t window_size);
+
 /**
  * Enables or disables a virtio-snd device.
  *
@@ -339,7 +373,6 @@ int32_t krun_add_vsock_port(uint32_t ctx_id,
  */
 int32_t krun_get_shutdown_eventfd(uint32_t ctx_id);
 
-
 /**
  * Configures the console device to ignore stdin and write the output to "c_filepath".
  *

src/devices/src/virtio/fs/device.rs

@@ -50,6 +50,7 @@ pub struct Fs {
     device_state: DeviceState,
     config: VirtioFsConfig,
     shm_region: Option<VirtioShmRegion>,
+    shm_size: u64,
     passthrough_cfg: passthrough::Config,
     worker_thread: Option<JoinHandle<()>>,
     worker_stopfd: EventFd,
@@ -62,6 +63,7 @@ impl Fs {
         fs_id: String,
         shared_dir: String,
         queues: Vec<VirtQueue>,
+        window_size: u64,
     ) -> super::Result<Fs> {
         let mut queue_events = Vec::new();
         for _ in 0..queues.len() {
@@ -93,6 +95,7 @@ impl Fs {
             device_state: DeviceState::Inactive,
             config,
             shm_region: None,
+            shm_size: window_size,
             passthrough_cfg: fs_cfg,
             worker_thread: None,
             worker_stopfd: EventFd::new(EFD_NONBLOCK).map_err(FsError::EventFd)?,
@@ -101,12 +104,12 @@ impl Fs {
         })
     }
 
-    pub fn new(fs_id: String, shared_dir: String) -> super::Result<Fs> {
+    pub fn new(fs_id: String, shared_dir: String, window_size: u64) -> super::Result<Fs> {
         let queues: Vec<VirtQueue> = defs::QUEUE_SIZES
             .iter()
             .map(|&max_size| VirtQueue::new(max_size))
             .collect();
-        Self::with_queues(fs_id, shared_dir, queues)
+        Self::with_queues(fs_id, shared_dir, queues, window_size)
     }
 
     pub fn id(&self) -> &str {
@@ -121,6 +124,10 @@ impl Fs {
         self.shm_region = Some(shm_region);
     }
 
+    pub fn get_shm_size(&self) -> u64 {
+        self.shm_size
+    }
+
     #[cfg(target_os = "macos")]
     pub fn set_map_sender(&mut self, map_sender: Sender<MemoryMapping>) {
         self.map_sender = Some(map_sender);

src/libkrun/src/lib.rs

@@ -96,6 +96,7 @@ struct ContextConfig {
     unix_ipc_port_map: Option<HashMap<u32, PathBuf>>,
     shutdown_efd: Option<EventFd>,
     gpu_virgl_flags: Option<u32>,
+    gpu_window_size: Option<u64>,
     enable_snd: bool,
     console_output: Option<PathBuf>,
 }
@@ -223,6 +224,10 @@ impl ContextConfig {
     fn set_gpu_virgl_flags(&mut self, virgl_flags: u32) {
         self.gpu_virgl_flags = Some(virgl_flags);
     }
+
+    fn set_gpu_window_size(&mut self, window_size: u64) {
+        self.gpu_window_size = Some(window_size);
+    }
 }
 
 static CTX_MAP: Lazy<Mutex<HashMap<u32, ContextConfig>>> = Lazy::new(|| Mutex::new(HashMap::new()));
@@ -398,7 +403,11 @@ pub unsafe extern "C" fn krun_set_root(ctx_id: u32, c_root_path: *const c_char)
             if !cfg.fs_devs.is_empty() {
                 return -libc::EINVAL;
             }
-            cfg.add_fs_dev(FsDeviceConfig { fs_id, shared_dir });
+            cfg.add_fs_dev(FsDeviceConfig {
+                fs_id,
+                shared_dir,
+                window_size: 0u64,
+            });
         }
         Entry::Vacant(_) => return -libc::ENOENT,
     }
@@ -429,6 +438,41 @@ pub unsafe extern "C" fn krun_add_virtiofs(
             cfg.add_fs_dev(FsDeviceConfig {
                 fs_id: tag.to_string(),
                 shared_dir: path.to_string(),
+                // Default to a conservative 512MB window
+                window_size: 1 << 29,
+            });
+        }
+        Entry::Vacant(_) => return -libc::ENOENT,
+    }
+
+    KRUN_SUCCESS
+}
+
+#[allow(clippy::missing_safety_doc)]
+#[no_mangle]
+#[cfg(not(feature = "tee"))]
+pub unsafe extern "C" fn krun_add_virtiofs2(
+    ctx_id: u32,
+    c_tag: *const c_char,
+    c_path: *const c_char,
+    window_size: u64,
+) -> i32 {
+    let tag = match CStr::from_ptr(c_tag).to_str() {
+        Ok(tag) => tag,
+        Err(_) => return -libc::EINVAL,
+    };
+    let path = match CStr::from_ptr(c_path).to_str() {
+        Ok(path) => path,
+        Err(_) => return -libc::EINVAL,
+    };
+
+    match CTX_MAP.lock().unwrap().entry(ctx_id) {
+        Entry::Occupied(mut ctx_cfg) => {
+            let cfg = ctx_cfg.get_mut();
+            cfg.add_fs_dev(FsDeviceConfig {
+                fs_id: tag.to_string(),
+                shared_dir: path.to_string(),
+                window_size,
             });
         }
         Entry::Vacant(_) => return -libc::ENOENT,
@@ -836,6 +880,25 @@ pub unsafe extern "C" fn krun_set_gpu_options(ctx_id: u32, virgl_flags: u32) ->
     KRUN_SUCCESS
 }
 
+#[allow(clippy::missing_safety_doc)]
+#[no_mangle]
+pub unsafe extern "C" fn krun_set_gpu_options2(
+    ctx_id: u32,
+    virgl_flags: u32,
+    window_size: u64,
+) -> i32 {
+    match CTX_MAP.lock().unwrap().entry(ctx_id) {
+        Entry::Occupied(mut ctx_cfg) => {
+            let cfg = ctx_cfg.get_mut();
+            cfg.set_gpu_virgl_flags(virgl_flags);
+            cfg.set_gpu_window_size(window_size);
+        }
+        Entry::Vacant(_) => return -libc::ENOENT,
+    }
+
+    KRUN_SUCCESS
+}
+
 #[allow(clippy::missing_safety_doc)]
 #[no_mangle]
 pub unsafe extern "C" fn krun_set_snd_device(ctx_id: u32, enable: bool) -> i32 {
@@ -1066,6 +1129,9 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
     if let Some(virgl_flags) = ctx_cfg.gpu_virgl_flags {
         ctx_cfg.vmr.set_gpu_virgl_flags(virgl_flags);
     }
+    if let Some(window_size) = ctx_cfg.gpu_window_size {
+        ctx_cfg.vmr.set_gpu_window_size(window_size);
+    }
 
     #[cfg(feature = "snd")]
     ctx_cfg.vmr.set_snd_device(ctx_cfg.enable_snd);

src/vmm/src/builder.rs

@@ -603,12 +603,15 @@ pub fn build_microvm(
     )?;
     #[cfg(feature = "gpu")]
     if let Some(virgl_flags) = vm_resources.gpu_virgl_flags {
+        // Default to 8GB, which was the value used in previous versions.
+        let window_size = vm_resources.gpu_window_size.unwrap_or(1 << 33);
         attach_gpu_device(
             &mut vmm,
             event_manager,
             &mut shm_manager,
             intc.clone(),
             virgl_flags,
+            window_size,
             #[cfg(target_os = "macos")]
             _map_sender.clone(),
         )?;
@@ -1142,7 +1145,7 @@ fn attach_fs_devices(
         }
 
         let shm_region = shm_manager
-            .get_region(1 << 30)
+            .get_region(fs.lock().unwrap().get_shm_size().try_into().unwrap())
             .map_err(StartMicrovmError::ShmCreate)?;
         let virtio_shm_region = vmm
             .add_shm_region(shm_region)
@@ -1407,6 +1410,7 @@ fn attach_gpu_device(
     shm_manager: &mut ShmManager,
     intc: Option<Arc<Mutex<Gic>>>,
     virgl_flags: u32,
+    window_size: u64,
     #[cfg(target_os = "macos")] map_sender: Sender<MemoryMapping>,
 ) -> std::result::Result<(), StartMicrovmError> {
     use self::StartMicrovmError::*;
@@ -1431,7 +1435,7 @@ fn attach_gpu_device(
     }
 
     let shm_region = shm_manager
-        .get_region(1 << 33)
+        .get_region(window_size.try_into().unwrap())
         .map_err(StartMicrovmError::ShmCreate)?;
     let virtio_shm_region = vmm
         .add_shm_region(shm_region)

src/vmm/src/resources.rs

@@ -110,6 +110,7 @@ pub struct VmResources {
     pub tee_config: TeeConfig,
     /// Flags for the virtio-gpu device.
     pub gpu_virgl_flags: Option<u32>,
+    pub gpu_window_size: Option<u64>,
     #[cfg(feature = "snd")]
     /// Enable the virtio-snd device.
     pub snd_device: bool,
@@ -254,6 +255,10 @@ impl VmResources {
         self.gpu_virgl_flags = Some(virgl_flags);
     }
 
+    pub fn set_gpu_window_size(&mut self, window_size: u64) {
+        self.gpu_window_size = Some(window_size);
+    }
+
     #[cfg(feature = "snd")]
     pub fn set_snd_device(&mut self, enabled: bool) {
         self.snd_device = enabled;
@@ -326,6 +331,7 @@ mod tests {
             #[cfg(feature = "net")]
             net_builder: Default::default(),
             gpu_virgl_flags: None,
+            gpu_window_size: None,
             #[cfg(feature = "snd")]
             enable_snd: False,
             console_output: None,

src/vmm/src/vmm_config/fs.rs

@@ -25,6 +25,7 @@ type Result<T> = std::result::Result<T, FsConfigError>;
 pub struct FsDeviceConfig {
     pub fs_id: String,
     pub shared_dir: String,
+    pub window_size: u64,
 }
 
 #[derive(Default)]
@@ -46,7 +47,7 @@ impl FsBuilder {
     }
 
     pub fn create_fs(config: FsDeviceConfig) -> Result<Fs> {
-        devices::virtio::Fs::new(config.fs_id, config.shared_dir)
+        devices::virtio::Fs::new(config.fs_id, config.shared_dir, config.window_size)
             .map_err(FsConfigError::CreateFsDevice)
     }
 }