From 4f5a4d59ce3a27da4732337042371b3b57998452 Mon Sep 17 00:00:00 2001 From: Sergio Lopez Date: Thu, 22 Aug 2024 10:40:08 +0200 Subject: [PATCH] libkrun: (gpu/fs) introduce variants set shm size Introduce krun_add_virtiofs2 and krun_set_gpu_options2 as variants of krun_add_virtiofs and krun_set_gpu_options respectively, to allow users to specify the DAX/host SHM window size for the device. We're adding variants of existing functions to avoid breaking existing consumers of the library. This isn't great, but we'll do a huge clean up of the API in 2.0. Signed-off-by: Sergio Lopez --- include/libkrun.h | 57 ++++++++++++++++++----- src/libkrun/src/lib.rs | 88 ++++++++++++++++++++++++++++-------- src/vmm/src/builder.rs | 26 +++++++---- src/vmm/src/resources.rs | 15 +++--- src/vmm/src/vmm_config/fs.rs | 50 +------------------- 5 files changed, 142 insertions(+), 94 deletions(-) diff --git a/include/libkrun.h b/include/libkrun.h index 15090769..41c3a57a 100644 --- a/include/libkrun.h +++ b/include/libkrun.h @@ -121,6 +121,24 @@ int32_t krun_add_virtiofs(uint32_t ctx_id, const char *c_tag, const char *c_path); +/** + * Adds an independent virtio-fs device pointing to a host's directory with a tag. This + * variant allows specifying the size of the DAX window. + * + * Arguments: + * "ctx_id" - the configuration context ID. + * "c_tag" - tag to identify the filesystem in the guest. + * "c_path" - full path to the directory in the host to be exposed to the guest. + * "shm_size" - size of the DAX SHM window in bytes. + * + * Returns: + * Zero on success or a negative error number on failure. + */ +int32_t krun_add_virtiofs2(uint32_t ctx_id, + const char *c_tag, + const char *c_path, + uint64_t shm_size); + /** * Configures the networking to use passt. * Call to this function disables TSI backend to use passt instead. @@ -154,7 +172,7 @@ int32_t krun_set_passt_fd(uint32_t ctx_id, int fd); * Returns: * Zero on success or a negative error number on failure. */ -int32_t krun_set_gvproxy_path(uint32_t ctx_id, char* c_path); +int32_t krun_set_gvproxy_path(uint32_t ctx_id, char *c_path); /** * Sets the MAC address for the virtio-net device when using the passt backend. @@ -196,17 +214,17 @@ int32_t krun_set_net_mac(uint32_t ctx_id, uint8_t *const c_mac); int32_t krun_set_port_map(uint32_t ctx_id, const char *const port_map[]); /* Flags for virglrenderer. Copied from virglrenderer bindings. */ -#define VIRGLRENDERER_USE_EGL 1 << 0 -#define VIRGLRENDERER_THREAD_SYNC 1 << 1 -#define VIRGLRENDERER_USE_GLX 1 << 2 -#define VIRGLRENDERER_USE_SURFACELESS 1 << 3 -#define VIRGLRENDERER_USE_GLES 1 << 4 -#define VIRGLRENDERER_USE_EXTERNAL_BLOB 1 << 5 -#define VIRGLRENDERER_VENUS 1 << 6 -#define VIRGLRENDERER_NO_VIRGL 1 << 7 +#define VIRGLRENDERER_USE_EGL 1 << 0 +#define VIRGLRENDERER_THREAD_SYNC 1 << 1 +#define VIRGLRENDERER_USE_GLX 1 << 2 +#define VIRGLRENDERER_USE_SURFACELESS 1 << 3 +#define VIRGLRENDERER_USE_GLES 1 << 4 +#define VIRGLRENDERER_USE_EXTERNAL_BLOB 1 << 5 +#define VIRGLRENDERER_VENUS 1 << 6 +#define VIRGLRENDERER_NO_VIRGL 1 << 7 #define VIRGLRENDERER_USE_ASYNC_FENCE_CB 1 << 8 -#define VIRGLRENDERER_RENDER_SERVER 1 << 9 -#define VIRGLRENDERER_DRM 1 << 10 +#define VIRGLRENDERER_RENDER_SERVER 1 << 9 +#define VIRGLRENDERER_DRM 1 << 10 /** * Enables and configures a virtio-gpu device. * @@ -219,6 +237,22 @@ int32_t krun_set_port_map(uint32_t ctx_id, const char *const port_map[]); */ int32_t krun_set_gpu_options(uint32_t ctx_id, uint32_t virgl_flags); +/** + * Enables and configures a virtio-gpu device. This variant allows specifying + * the size of the host window (acting as vRAM in the guest). + * + * Arguments: + * "ctx_id" - the configuration context ID. + * "virgl_flags" - flags to pass to virglrenderer. + * "shm_size" - size of the SHM host window in bytes. + * + * Returns: + * Zero on success or a negative error number on failure. + */ +int32_t krun_set_gpu_options2(uint32_t ctx_id, + uint32_t virgl_flags, + uint64_t shm_size); + /** * Enables or disables a virtio-snd device. * @@ -339,7 +373,6 @@ int32_t krun_add_vsock_port(uint32_t ctx_id, */ int32_t krun_get_shutdown_eventfd(uint32_t ctx_id); - /** * Configures the console device to ignore stdin and write the output to "c_filepath". * diff --git a/src/libkrun/src/lib.rs b/src/libkrun/src/lib.rs index 6cd2b156..ec6f2ddd 100644 --- a/src/libkrun/src/lib.rs +++ b/src/libkrun/src/lib.rs @@ -85,8 +85,6 @@ struct ContextConfig { rlimits: Option, net_cfg: NetworkConfig, mac: Option<[u8; 6]>, - #[cfg(not(feature = "tee"))] - fs_devs: Vec, #[cfg(feature = "blk")] root_block_cfg: Option, #[cfg(feature = "blk")] @@ -96,6 +94,7 @@ struct ContextConfig { unix_ipc_port_map: Option>, shutdown_efd: Option, gpu_virgl_flags: Option, + gpu_shm_size: Option, enable_snd: bool, console_output: Option, } @@ -156,11 +155,6 @@ impl ContextConfig { } } - #[cfg(not(feature = "tee"))] - fn add_fs_dev(&mut self, fs_cfg: FsDeviceConfig) { - self.fs_devs.push(fs_cfg) - } - #[cfg(feature = "blk")] fn set_root_block_cfg(&mut self, block_cfg: BlockDeviceConfig) { self.root_block_cfg = Some(block_cfg); @@ -223,6 +217,10 @@ impl ContextConfig { fn set_gpu_virgl_flags(&mut self, virgl_flags: u32) { self.gpu_virgl_flags = Some(virgl_flags); } + + fn set_gpu_shm_size(&mut self, shm_size: usize) { + self.gpu_shm_size = Some(shm_size); + } } static CTX_MAP: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); @@ -395,10 +393,12 @@ pub unsafe extern "C" fn krun_set_root(ctx_id: u32, c_root_path: *const c_char) match CTX_MAP.lock().unwrap().entry(ctx_id) { Entry::Occupied(mut ctx_cfg) => { let cfg = ctx_cfg.get_mut(); - if !cfg.fs_devs.is_empty() { - return -libc::EINVAL; - } - cfg.add_fs_dev(FsDeviceConfig { fs_id, shared_dir }); + cfg.vmr.add_fs_device(FsDeviceConfig { + fs_id, + shared_dir, + // Default to a conservative 512 MB window. + shm_size: Some(1 << 29), + }); } Entry::Vacant(_) => return -libc::ENOENT, } @@ -426,9 +426,43 @@ pub unsafe extern "C" fn krun_add_virtiofs( match CTX_MAP.lock().unwrap().entry(ctx_id) { Entry::Occupied(mut ctx_cfg) => { let cfg = ctx_cfg.get_mut(); - cfg.add_fs_dev(FsDeviceConfig { + cfg.vmr.add_fs_device(FsDeviceConfig { + fs_id: tag.to_string(), + shared_dir: path.to_string(), + shm_size: None, + }); + } + Entry::Vacant(_) => return -libc::ENOENT, + } + + KRUN_SUCCESS +} + +#[allow(clippy::missing_safety_doc)] +#[no_mangle] +#[cfg(not(feature = "tee"))] +pub unsafe extern "C" fn krun_add_virtiofs2( + ctx_id: u32, + c_tag: *const c_char, + c_path: *const c_char, + shm_size: u64, +) -> i32 { + let tag = match CStr::from_ptr(c_tag).to_str() { + Ok(tag) => tag, + Err(_) => return -libc::EINVAL, + }; + let path = match CStr::from_ptr(c_path).to_str() { + Ok(path) => path, + Err(_) => return -libc::EINVAL, + }; + + match CTX_MAP.lock().unwrap().entry(ctx_id) { + Entry::Occupied(mut ctx_cfg) => { + let cfg = ctx_cfg.get_mut(); + cfg.vmr.add_fs_device(FsDeviceConfig { fs_id: tag.to_string(), shared_dir: path.to_string(), + shm_size: Some(shm_size.try_into().unwrap()), }); } Entry::Vacant(_) => return -libc::ENOENT, @@ -836,6 +870,25 @@ pub unsafe extern "C" fn krun_set_gpu_options(ctx_id: u32, virgl_flags: u32) -> KRUN_SUCCESS } +#[allow(clippy::missing_safety_doc)] +#[no_mangle] +pub unsafe extern "C" fn krun_set_gpu_options2( + ctx_id: u32, + virgl_flags: u32, + shm_size: u64, +) -> i32 { + match CTX_MAP.lock().unwrap().entry(ctx_id) { + Entry::Occupied(mut ctx_cfg) => { + let cfg = ctx_cfg.get_mut(); + cfg.set_gpu_virgl_flags(virgl_flags); + cfg.set_gpu_shm_size(shm_size.try_into().unwrap()); + } + Entry::Vacant(_) => return -libc::ENOENT, + } + + KRUN_SUCCESS +} + #[allow(clippy::missing_safety_doc)] #[no_mangle] pub unsafe extern "C" fn krun_set_snd_device(ctx_id: u32, enable: bool) -> i32 { @@ -967,14 +1020,6 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 { None => return -libc::ENOENT, }; - #[cfg(not(feature = "tee"))] - for fs in &ctx_cfg.fs_devs { - if ctx_cfg.vmr.add_fs_device(fs.clone()).is_err() { - error!("Error configuring virtio-fs"); - return -libc::EINVAL; - } - } - #[cfg(feature = "blk")] if let Some(block_cfg) = ctx_cfg.get_root_block_cfg() { if ctx_cfg.vmr.add_block_device(block_cfg).is_err() { @@ -1066,6 +1111,9 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 { if let Some(virgl_flags) = ctx_cfg.gpu_virgl_flags { ctx_cfg.vmr.set_gpu_virgl_flags(virgl_flags); } + if let Some(shm_size) = ctx_cfg.gpu_shm_size { + ctx_cfg.vmr.set_gpu_shm_size(shm_size); + } #[cfg(feature = "snd")] ctx_cfg.vmr.set_snd_device(ctx_cfg.enable_snd); diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index d1b950d8..3b95d0ec 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -43,7 +43,10 @@ use crate::terminal::term_set_raw_mode; use crate::vmm_config::block::BlockBuilder; use crate::vmm_config::boot_source::DEFAULT_KERNEL_CMDLINE; #[cfg(not(feature = "tee"))] -use crate::vmm_config::fs::FsBuilder; +use crate::vmm_config::fs::FsDeviceConfig; +#[cfg(feature = "tee")] +use crate::vmm_config::kernel_bundle::{InitrdBundle, QbootBundle}; +>>>>>>> libkrun: (gpu/fs) introduce variants set shm size #[cfg(target_os = "linux")] use crate::vstate::KvmContext; #[cfg(all(target_os = "linux", feature = "tee"))] @@ -842,14 +845,17 @@ fn create_guest_memory( if let Some(vm_resources) = vm_resources { #[cfg(not(feature = "tee"))] - for (index, _fs) in vm_resources.fs.list.iter().enumerate() { - shm_manager - .create_fs_region(index, 1 << 29) - .map_err(StartMicrovmError::ShmCreate)?; + for (index, fs) in vm_resources.fs.iter().enumerate() { + if let Some(shm_size) = fs.shm_size { + shm_manager + .create_fs_region(index, shm_size) + .map_err(StartMicrovmError::ShmCreate)?; + } } if vm_resources.gpu_virgl_flags.is_some() { + let size = vm_resources.gpu_shm_size.unwrap_or(1 << 33); shm_manager - .create_gpu_region(1 << 33) + .create_gpu_region(size) .map_err(StartMicrovmError::ShmCreate)?; } @@ -1177,14 +1183,18 @@ fn attach_mmio_device( #[cfg(not(feature = "tee"))] fn attach_fs_devices( vmm: &mut Vmm, - fs_devs: &FsBuilder, + fs_devs: &[FsDeviceConfig], shm_manager: &mut ShmManager, intc: Option>>, #[cfg(target_os = "macos")] map_sender: Sender, ) -> std::result::Result<(), StartMicrovmError> { use self::StartMicrovmError::*; - for (i, fs) in fs_devs.list.iter().enumerate() { + for (i, config) in fs_devs.iter().enumerate() { + let fs = Arc::new(Mutex::new( + devices::virtio::Fs::new(config.fs_id.clone(), config.shared_dir.clone()).unwrap(), + )); + let id = format!("{}{}", String::from(fs.lock().unwrap().id()), i); if let Some(ref intc) = intc { diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs index f8bd6571..35be6461 100644 --- a/src/vmm/src/resources.rs +++ b/src/vmm/src/resources.rs @@ -41,9 +41,6 @@ pub enum Error { /// Error opening TEE config file. #[cfg(feature = "tee")] OpenTeeConfig(std::io::Error), - /// Fs device configuration error. - #[cfg(not(feature = "tee"))] - FsDevice(FsConfigError), /// Error parsing TEE config file. #[cfg(feature = "tee")] ParseTeeConfig(serde_json::Error), @@ -96,7 +93,7 @@ pub struct VmResources { pub initrd_bundle: Option, /// The fs device. #[cfg(not(feature = "tee"))] - pub fs: FsBuilder, + pub fs: Vec, /// The vsock device. pub vsock: VsockBuilder, /// The virtio-blk device. @@ -110,6 +107,7 @@ pub struct VmResources { pub tee_config: TeeConfig, /// Flags for the virtio-gpu device. pub gpu_virgl_flags: Option, + pub gpu_shm_size: Option, #[cfg(feature = "snd")] /// Enable the virtio-snd device. pub snd_device: bool, @@ -236,8 +234,8 @@ impl VmResources { } #[cfg(not(feature = "tee"))] - pub fn add_fs_device(&mut self, config: FsDeviceConfig) -> Result { - self.fs.insert(config) + pub fn add_fs_device(&mut self, config: FsDeviceConfig) { + self.fs.push(config) } #[cfg(feature = "blk")] @@ -254,6 +252,10 @@ impl VmResources { self.gpu_virgl_flags = Some(virgl_flags); } + pub fn set_gpu_shm_size(&mut self, shm_size: usize) { + self.gpu_shm_size = Some(shm_size); + } + #[cfg(feature = "snd")] pub fn set_snd_device(&mut self, enabled: bool) { self.snd_device = enabled; @@ -326,6 +328,7 @@ mod tests { #[cfg(feature = "net")] net_builder: Default::default(), gpu_virgl_flags: None, + gpu_shm_size: None, #[cfg(feature = "snd")] enable_snd: False, console_output: None, diff --git a/src/vmm/src/vmm_config/fs.rs b/src/vmm/src/vmm_config/fs.rs index ba16df39..cc799502 100644 --- a/src/vmm/src/vmm_config/fs.rs +++ b/src/vmm/src/vmm_config/fs.rs @@ -1,52 +1,6 @@ -use std::collections::VecDeque; -use std::fmt; -use std::sync::{Arc, Mutex}; - -use devices::virtio::{Fs, FsError}; - -#[derive(Debug)] -pub enum FsConfigError { - /// Failed to create the fs device. - CreateFsDevice(FsError), -} - -impl fmt::Display for FsConfigError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::FsConfigError::*; - match *self { - CreateFsDevice(ref e) => write!(f, "Cannot create vsock device: {e:?}"), - } - } -} - -type Result = std::result::Result; - -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug)] pub struct FsDeviceConfig { pub fs_id: String, pub shared_dir: String, -} - -#[derive(Default)] -pub struct FsBuilder { - pub list: VecDeque>>, -} - -impl FsBuilder { - pub fn new() -> Self { - Self { - list: VecDeque::>>::new(), - } - } - - pub fn insert(&mut self, config: FsDeviceConfig) -> Result<()> { - let fs_dev = Arc::new(Mutex::new(Self::create_fs(config)?)); - self.list.push_back(fs_dev); - Ok(()) - } - - pub fn create_fs(config: FsDeviceConfig) -> Result { - devices::virtio::Fs::new(config.fs_id, config.shared_dir) - .map_err(FsConfigError::CreateFsDevice) - } + pub shm_size: Option, }