From 5ad66f5d8f8889c1f65dd91351912a0c34c17ac3 Mon Sep 17 00:00:00 2001 From: Sergio Lopez Date: Thu, 22 Aug 2024 10:40:08 +0200 Subject: [PATCH] libkrun: (gpu/fs) introduce variants set win size Introduce krun_add_virtiofs2 and krun_set_gpu_options2 as variants of krun_add_virtiofs and krun_set_gpu_options respectively, to allow users to specify the DAX/host window size for the device. We're adding variants of existing functions to avoid breaking existing consumers of the library. This isn't great, but we'll do a huge clean up of the API in 2.0. Signed-off-by: Sergio Lopez --- include/libkrun.h | 37 +++++++++++++++- src/devices/src/virtio/fs/device.rs | 11 ++++- src/libkrun/src/lib.rs | 68 ++++++++++++++++++++++++++++- src/vmm/src/builder.rs | 8 +++- src/vmm/src/resources.rs | 6 +++ src/vmm/src/vmm_config/fs.rs | 3 +- 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/include/libkrun.h b/include/libkrun.h index 15090769..3cc86bc9 100644 --- a/include/libkrun.h +++ b/include/libkrun.h @@ -121,6 +121,24 @@ int32_t krun_add_virtiofs(uint32_t ctx_id, const char *c_tag, const char *c_path); +/** + * Adds an independent virtio-fs device pointing to a host's directory with a tag. This + * variant allows specifying the size of the DAX window. + * + * Arguments: + * "ctx_id" - the configuration context ID. + * "c_tag" - tag to identify the filesystem in the guest. + * "c_path" - full path to the directory in the host to be exposed to the guest. + * "window_size" - size of the DAX window in bytes. + * + * Returns: + * Zero on success or a negative error number on failure. + */ +int32_t krun_add_virtiofs2(uint32_t ctx_id, + const char *c_tag, + const char *c_path, + uint64_t window_size); + /** * Configures the networking to use passt. * Call to this function disables TSI backend to use passt instead. @@ -154,7 +172,7 @@ int32_t krun_set_passt_fd(uint32_t ctx_id, int fd); * Returns: * Zero on success or a negative error number on failure. */ -int32_t krun_set_gvproxy_path(uint32_t ctx_id, char* c_path); +int32_t krun_set_gvproxy_path(uint32_t ctx_id, char *c_path); /** * Sets the MAC address for the virtio-net device when using the passt backend. @@ -219,6 +237,22 @@ int32_t krun_set_port_map(uint32_t ctx_id, const char *const port_map[]); */ int32_t krun_set_gpu_options(uint32_t ctx_id, uint32_t virgl_flags); +/** + * Enables and configures a virtio-gpu device. This variant allows specifying + * the size of the host window (acting as vRAM in the guest). + * + * Arguments: + * "ctx_id" - the configuration context ID. + * "virgl_flags" - flags to pass to virglrenderer. + * "window_size" - size of the host window in bytes. + * + * Returns: + * Zero on success or a negative error number on failure. + */ +int32_t krun_set_gpu_options2(uint32_t ctx_id, + uint32_t virgl_flags, + uint64_t window_size); + /** * Enables or disables a virtio-snd device. * @@ -339,7 +373,6 @@ int32_t krun_add_vsock_port(uint32_t ctx_id, */ int32_t krun_get_shutdown_eventfd(uint32_t ctx_id); - /** * Configures the console device to ignore stdin and write the output to "c_filepath". * diff --git a/src/devices/src/virtio/fs/device.rs b/src/devices/src/virtio/fs/device.rs index 71ccd806..05a33afb 100644 --- a/src/devices/src/virtio/fs/device.rs +++ b/src/devices/src/virtio/fs/device.rs @@ -50,6 +50,7 @@ pub struct Fs { device_state: DeviceState, config: VirtioFsConfig, shm_region: Option, + shm_size: u64, passthrough_cfg: passthrough::Config, worker_thread: Option>, worker_stopfd: EventFd, @@ -62,6 +63,7 @@ impl Fs { fs_id: String, shared_dir: String, queues: Vec, + window_size: u64, ) -> super::Result { let mut queue_events = Vec::new(); for _ in 0..queues.len() { @@ -93,6 +95,7 @@ impl Fs { device_state: DeviceState::Inactive, config, shm_region: None, + shm_size: window_size, passthrough_cfg: fs_cfg, worker_thread: None, worker_stopfd: EventFd::new(EFD_NONBLOCK).map_err(FsError::EventFd)?, @@ -101,12 +104,12 @@ impl Fs { }) } - pub fn new(fs_id: String, shared_dir: String) -> super::Result { + pub fn new(fs_id: String, shared_dir: String, window_size: u64) -> super::Result { let queues: Vec = defs::QUEUE_SIZES .iter() .map(|&max_size| VirtQueue::new(max_size)) .collect(); - Self::with_queues(fs_id, shared_dir, queues) + Self::with_queues(fs_id, shared_dir, queues, window_size) } pub fn id(&self) -> &str { @@ -121,6 +124,10 @@ impl Fs { self.shm_region = Some(shm_region); } + pub fn get_shm_size(&self) -> u64 { + self.shm_size + } + #[cfg(target_os = "macos")] pub fn set_map_sender(&mut self, map_sender: Sender) { self.map_sender = Some(map_sender); diff --git a/src/libkrun/src/lib.rs b/src/libkrun/src/lib.rs index 6cd2b156..356f5e59 100644 --- a/src/libkrun/src/lib.rs +++ b/src/libkrun/src/lib.rs @@ -96,6 +96,7 @@ struct ContextConfig { unix_ipc_port_map: Option>, shutdown_efd: Option, gpu_virgl_flags: Option, + gpu_window_size: Option, enable_snd: bool, console_output: Option, } @@ -223,6 +224,10 @@ impl ContextConfig { fn set_gpu_virgl_flags(&mut self, virgl_flags: u32) { self.gpu_virgl_flags = Some(virgl_flags); } + + fn set_gpu_window_size(&mut self, window_size: u64) { + self.gpu_window_size = Some(window_size); + } } static CTX_MAP: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); @@ -398,7 +403,11 @@ pub unsafe extern "C" fn krun_set_root(ctx_id: u32, c_root_path: *const c_char) if !cfg.fs_devs.is_empty() { return -libc::EINVAL; } - cfg.add_fs_dev(FsDeviceConfig { fs_id, shared_dir }); + cfg.add_fs_dev(FsDeviceConfig { + fs_id, + shared_dir, + window_size: 0u64, + }); } Entry::Vacant(_) => return -libc::ENOENT, } @@ -429,6 +438,41 @@ pub unsafe extern "C" fn krun_add_virtiofs( cfg.add_fs_dev(FsDeviceConfig { fs_id: tag.to_string(), shared_dir: path.to_string(), + // Default to a conservative 512MB window + window_size: 1 << 29, + }); + } + Entry::Vacant(_) => return -libc::ENOENT, + } + + KRUN_SUCCESS +} + +#[allow(clippy::missing_safety_doc)] +#[no_mangle] +#[cfg(not(feature = "tee"))] +pub unsafe extern "C" fn krun_add_virtiofs2( + ctx_id: u32, + c_tag: *const c_char, + c_path: *const c_char, + window_size: u64, +) -> i32 { + let tag = match CStr::from_ptr(c_tag).to_str() { + Ok(tag) => tag, + Err(_) => return -libc::EINVAL, + }; + let path = match CStr::from_ptr(c_path).to_str() { + Ok(path) => path, + Err(_) => return -libc::EINVAL, + }; + + match CTX_MAP.lock().unwrap().entry(ctx_id) { + Entry::Occupied(mut ctx_cfg) => { + let cfg = ctx_cfg.get_mut(); + cfg.add_fs_dev(FsDeviceConfig { + fs_id: tag.to_string(), + shared_dir: path.to_string(), + window_size, }); } Entry::Vacant(_) => return -libc::ENOENT, @@ -836,6 +880,25 @@ pub unsafe extern "C" fn krun_set_gpu_options(ctx_id: u32, virgl_flags: u32) -> KRUN_SUCCESS } +#[allow(clippy::missing_safety_doc)] +#[no_mangle] +pub unsafe extern "C" fn krun_set_gpu_options2( + ctx_id: u32, + virgl_flags: u32, + window_size: u64, +) -> i32 { + match CTX_MAP.lock().unwrap().entry(ctx_id) { + Entry::Occupied(mut ctx_cfg) => { + let cfg = ctx_cfg.get_mut(); + cfg.set_gpu_virgl_flags(virgl_flags); + cfg.set_gpu_window_size(window_size); + } + Entry::Vacant(_) => return -libc::ENOENT, + } + + KRUN_SUCCESS +} + #[allow(clippy::missing_safety_doc)] #[no_mangle] pub unsafe extern "C" fn krun_set_snd_device(ctx_id: u32, enable: bool) -> i32 { @@ -1066,6 +1129,9 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 { if let Some(virgl_flags) = ctx_cfg.gpu_virgl_flags { ctx_cfg.vmr.set_gpu_virgl_flags(virgl_flags); } + if let Some(window_size) = ctx_cfg.gpu_window_size { + ctx_cfg.vmr.set_gpu_window_size(window_size); + } #[cfg(feature = "snd")] ctx_cfg.vmr.set_snd_device(ctx_cfg.enable_snd); diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 6bade51f..78d13162 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -603,12 +603,15 @@ pub fn build_microvm( )?; #[cfg(feature = "gpu")] if let Some(virgl_flags) = vm_resources.gpu_virgl_flags { + // Default to 8GB, which was the value used in previous versions. + let window_size = vm_resources.gpu_window_size.unwrap_or(1 << 33); attach_gpu_device( &mut vmm, event_manager, &mut shm_manager, intc.clone(), virgl_flags, + window_size, #[cfg(target_os = "macos")] _map_sender.clone(), )?; @@ -1142,7 +1145,7 @@ fn attach_fs_devices( } let shm_region = shm_manager - .get_region(1 << 30) + .get_region(fs.lock().unwrap().get_shm_size().try_into().unwrap()) .map_err(StartMicrovmError::ShmCreate)?; let virtio_shm_region = vmm .add_shm_region(shm_region) @@ -1407,6 +1410,7 @@ fn attach_gpu_device( shm_manager: &mut ShmManager, intc: Option>>, virgl_flags: u32, + window_size: u64, #[cfg(target_os = "macos")] map_sender: Sender, ) -> std::result::Result<(), StartMicrovmError> { use self::StartMicrovmError::*; @@ -1431,7 +1435,7 @@ fn attach_gpu_device( } let shm_region = shm_manager - .get_region(1 << 33) + .get_region(window_size.try_into().unwrap()) .map_err(StartMicrovmError::ShmCreate)?; let virtio_shm_region = vmm .add_shm_region(shm_region) diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs index f8bd6571..e057121c 100644 --- a/src/vmm/src/resources.rs +++ b/src/vmm/src/resources.rs @@ -110,6 +110,7 @@ pub struct VmResources { pub tee_config: TeeConfig, /// Flags for the virtio-gpu device. pub gpu_virgl_flags: Option, + pub gpu_window_size: Option, #[cfg(feature = "snd")] /// Enable the virtio-snd device. pub snd_device: bool, @@ -254,6 +255,10 @@ impl VmResources { self.gpu_virgl_flags = Some(virgl_flags); } + pub fn set_gpu_window_size(&mut self, window_size: u64) { + self.gpu_window_size = Some(window_size); + } + #[cfg(feature = "snd")] pub fn set_snd_device(&mut self, enabled: bool) { self.snd_device = enabled; @@ -326,6 +331,7 @@ mod tests { #[cfg(feature = "net")] net_builder: Default::default(), gpu_virgl_flags: None, + gpu_window_size: None, #[cfg(feature = "snd")] enable_snd: False, console_output: None, diff --git a/src/vmm/src/vmm_config/fs.rs b/src/vmm/src/vmm_config/fs.rs index ba16df39..62b033d1 100644 --- a/src/vmm/src/vmm_config/fs.rs +++ b/src/vmm/src/vmm_config/fs.rs @@ -25,6 +25,7 @@ type Result = std::result::Result; pub struct FsDeviceConfig { pub fs_id: String, pub shared_dir: String, + pub window_size: u64, } #[derive(Default)] @@ -46,7 +47,7 @@ impl FsBuilder { } pub fn create_fs(config: FsDeviceConfig) -> Result { - devices::virtio::Fs::new(config.fs_id, config.shared_dir) + devices::virtio::Fs::new(config.fs_id, config.shared_dir, config.window_size) .map_err(FsConfigError::CreateFsDevice) } }