Skip to content

Commit

Permalink
libkrun: (gpu/fs) introduce variants set shm size
Browse files Browse the repository at this point in the history 
Introduce krun_add_virtiofs2 and krun_set_gpu_options2 as variants of
krun_add_virtiofs and krun_set_gpu_options respectively, to allow users
to specify the DAX/host SHM window size for the device.

We're adding variants of existing functions to avoid breaking existing
consumers of the library. This isn't great, but we'll do a huge clean up
of the API in 2.0.

Signed-off-by: Sergio Lopez <[email protected]>
  • Loading branch information
@slp @tylerfanelli
slp authored and tylerfanelli committed Sep 5, 2024
1 parent a156f1e commit 7b13b39
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 94 deletions.
57 changes: 45 additions & 12 deletions include/libkrun.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,24 @@ int32_t krun_add_virtiofs(uint32_t ctx_id,
const char *c_tag,
const char *c_path);

/**
* Adds an independent virtio-fs device pointing to a host's directory with a tag. This
* variant allows specifying the size of the DAX window.
*
* Arguments:
* "ctx_id" - the configuration context ID.
* "c_tag" - tag to identify the filesystem in the guest.
* "c_path" - full path to the directory in the host to be exposed to the guest.
* "shm_size" - size of the DAX SHM window in bytes.
*
* Returns:
* Zero on success or a negative error number on failure.
*/
int32_t krun_add_virtiofs2(uint32_t ctx_id,
const char *c_tag,
const char *c_path,
uint64_t shm_size);

/**
* Configures the networking to use passt.
* Call to this function disables TSI backend to use passt instead.
Expand Down Expand Up @@ -154,7 +172,7 @@ int32_t krun_set_passt_fd(uint32_t ctx_id, int fd);
* Returns:
* Zero on success or a negative error number on failure.
*/
int32_t krun_set_gvproxy_path(uint32_t ctx_id, char* c_path);
int32_t krun_set_gvproxy_path(uint32_t ctx_id, char *c_path);

/**
* Sets the MAC address for the virtio-net device when using the passt backend.
Expand Down Expand Up @@ -196,17 +214,17 @@ int32_t krun_set_net_mac(uint32_t ctx_id, uint8_t *const c_mac);
int32_t krun_set_port_map(uint32_t ctx_id, const char *const port_map[]);

/* Flags for virglrenderer. Copied from virglrenderer bindings. */
#define VIRGLRENDERER_USE_EGL 1 << 0
#define VIRGLRENDERER_THREAD_SYNC 1 << 1
#define VIRGLRENDERER_USE_GLX 1 << 2
#define VIRGLRENDERER_USE_SURFACELESS 1 << 3
#define VIRGLRENDERER_USE_GLES 1 << 4
#define VIRGLRENDERER_USE_EXTERNAL_BLOB 1 << 5
#define VIRGLRENDERER_VENUS 1 << 6
#define VIRGLRENDERER_NO_VIRGL 1 << 7
#define VIRGLRENDERER_USE_EGL 1 << 0
#define VIRGLRENDERER_THREAD_SYNC 1 << 1
#define VIRGLRENDERER_USE_GLX 1 << 2
#define VIRGLRENDERER_USE_SURFACELESS 1 << 3
#define VIRGLRENDERER_USE_GLES 1 << 4
#define VIRGLRENDERER_USE_EXTERNAL_BLOB 1 << 5
#define VIRGLRENDERER_VENUS 1 << 6
#define VIRGLRENDERER_NO_VIRGL 1 << 7
#define VIRGLRENDERER_USE_ASYNC_FENCE_CB 1 << 8
#define VIRGLRENDERER_RENDER_SERVER 1 << 9
#define VIRGLRENDERER_DRM 1 << 10
#define VIRGLRENDERER_RENDER_SERVER 1 << 9
#define VIRGLRENDERER_DRM 1 << 10
/**
* Enables and configures a virtio-gpu device.
*
Expand All @@ -219,6 +237,22 @@ int32_t krun_set_port_map(uint32_t ctx_id, const char *const port_map[]);
*/
int32_t krun_set_gpu_options(uint32_t ctx_id, uint32_t virgl_flags);

/**
* Enables and configures a virtio-gpu device. This variant allows specifying
* the size of the host window (acting as vRAM in the guest).
*
* Arguments:
* "ctx_id" - the configuration context ID.
* "virgl_flags" - flags to pass to virglrenderer.
* "shm_size" - size of the SHM host window in bytes.
*
* Returns:
* Zero on success or a negative error number on failure.
*/
int32_t krun_set_gpu_options2(uint32_t ctx_id,
uint32_t virgl_flags,
uint64_t shm_size);

/**
* Enables or disables a virtio-snd device.
*
Expand Down Expand Up @@ -339,7 +373,6 @@ int32_t krun_add_vsock_port(uint32_t ctx_id,
*/
int32_t krun_get_shutdown_eventfd(uint32_t ctx_id);


/**
* Configures the console device to ignore stdin and write the output to "c_filepath".
*
Expand Down
88 changes: 68 additions & 20 deletions src/libkrun/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ struct ContextConfig {
rlimits: Option<String>,
net_cfg: NetworkConfig,
mac: Option<[u8; 6]>,
#[cfg(not(feature = "tee"))]
fs_devs: Vec<FsDeviceConfig>,
#[cfg(feature = "blk")]
root_block_cfg: Option<BlockDeviceConfig>,
#[cfg(feature = "blk")]
Expand All @@ -96,6 +94,7 @@ struct ContextConfig {
unix_ipc_port_map: Option<HashMap<u32, PathBuf>>,
shutdown_efd: Option<EventFd>,
gpu_virgl_flags: Option<u32>,
gpu_shm_size: Option<usize>,
enable_snd: bool,
console_output: Option<PathBuf>,
}
Expand Down Expand Up @@ -156,11 +155,6 @@ impl ContextConfig {
}
}

#[cfg(not(feature = "tee"))]
fn add_fs_dev(&mut self, fs_cfg: FsDeviceConfig) {
self.fs_devs.push(fs_cfg)
}

#[cfg(feature = "blk")]
fn set_root_block_cfg(&mut self, block_cfg: BlockDeviceConfig) {
self.root_block_cfg = Some(block_cfg);
Expand Down Expand Up @@ -223,6 +217,10 @@ impl ContextConfig {
fn set_gpu_virgl_flags(&mut self, virgl_flags: u32) {
self.gpu_virgl_flags = Some(virgl_flags);
}

fn set_gpu_shm_size(&mut self, shm_size: usize) {
self.gpu_shm_size = Some(shm_size);
}
}

static CTX_MAP: Lazy<Mutex<HashMap<u32, ContextConfig>>> = Lazy::new(|| Mutex::new(HashMap::new()));
Expand Down Expand Up @@ -395,10 +393,12 @@ pub unsafe extern "C" fn krun_set_root(ctx_id: u32, c_root_path: *const c_char)
match CTX_MAP.lock().unwrap().entry(ctx_id) {
Entry::Occupied(mut ctx_cfg) => {
let cfg = ctx_cfg.get_mut();
if !cfg.fs_devs.is_empty() {
return -libc::EINVAL;
}
cfg.add_fs_dev(FsDeviceConfig { fs_id, shared_dir });
cfg.vmr.add_fs_device(FsDeviceConfig {
fs_id,
shared_dir,
// Default to a conservative 512 MB window.
shm_size: Some(1 << 29),
});
}
Entry::Vacant(_) => return -libc::ENOENT,
}
Expand Down Expand Up @@ -426,9 +426,43 @@ pub unsafe extern "C" fn krun_add_virtiofs(
match CTX_MAP.lock().unwrap().entry(ctx_id) {
Entry::Occupied(mut ctx_cfg) => {
let cfg = ctx_cfg.get_mut();
cfg.add_fs_dev(FsDeviceConfig {
cfg.vmr.add_fs_device(FsDeviceConfig {
fs_id: tag.to_string(),
shared_dir: path.to_string(),
shm_size: None,
});
}
Entry::Vacant(_) => return -libc::ENOENT,
}

KRUN_SUCCESS
}

#[allow(clippy::missing_safety_doc)]
#[no_mangle]
#[cfg(not(feature = "tee"))]
pub unsafe extern "C" fn krun_add_virtiofs2(
ctx_id: u32,
c_tag: *const c_char,
c_path: *const c_char,
shm_size: u64,
) -> i32 {
let tag = match CStr::from_ptr(c_tag).to_str() {
Ok(tag) => tag,
Err(_) => return -libc::EINVAL,
};
let path = match CStr::from_ptr(c_path).to_str() {
Ok(path) => path,
Err(_) => return -libc::EINVAL,
};

match CTX_MAP.lock().unwrap().entry(ctx_id) {
Entry::Occupied(mut ctx_cfg) => {
let cfg = ctx_cfg.get_mut();
cfg.vmr.add_fs_device(FsDeviceConfig {
fs_id: tag.to_string(),
shared_dir: path.to_string(),
shm_size: Some(shm_size.try_into().unwrap()),
});
}
Entry::Vacant(_) => return -libc::ENOENT,
Expand Down Expand Up @@ -836,6 +870,25 @@ pub unsafe extern "C" fn krun_set_gpu_options(ctx_id: u32, virgl_flags: u32) ->
KRUN_SUCCESS
}

#[allow(clippy::missing_safety_doc)]
#[no_mangle]
pub unsafe extern "C" fn krun_set_gpu_options2(
ctx_id: u32,
virgl_flags: u32,
shm_size: u64,
) -> i32 {
match CTX_MAP.lock().unwrap().entry(ctx_id) {
Entry::Occupied(mut ctx_cfg) => {
let cfg = ctx_cfg.get_mut();
cfg.set_gpu_virgl_flags(virgl_flags);
cfg.set_gpu_shm_size(shm_size.try_into().unwrap());
}
Entry::Vacant(_) => return -libc::ENOENT,
}

KRUN_SUCCESS
}

#[allow(clippy::missing_safety_doc)]
#[no_mangle]
pub unsafe extern "C" fn krun_set_snd_device(ctx_id: u32, enable: bool) -> i32 {
Expand Down Expand Up @@ -967,14 +1020,6 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
None => return -libc::ENOENT,
};

#[cfg(not(feature = "tee"))]
for fs in &ctx_cfg.fs_devs {
if ctx_cfg.vmr.add_fs_device(fs.clone()).is_err() {
error!("Error configuring virtio-fs");
return -libc::EINVAL;
}
}

#[cfg(feature = "blk")]
if let Some(block_cfg) = ctx_cfg.get_root_block_cfg() {
if ctx_cfg.vmr.add_block_device(block_cfg).is_err() {
Expand Down Expand Up @@ -1066,6 +1111,9 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
if let Some(virgl_flags) = ctx_cfg.gpu_virgl_flags {
ctx_cfg.vmr.set_gpu_virgl_flags(virgl_flags);
}
if let Some(shm_size) = ctx_cfg.gpu_shm_size {
ctx_cfg.vmr.set_gpu_shm_size(shm_size);
}

#[cfg(feature = "snd")]
ctx_cfg.vmr.set_snd_device(ctx_cfg.enable_snd);
Expand Down
23 changes: 15 additions & 8 deletions src/vmm/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ use crate::terminal::term_set_raw_mode;
use crate::vmm_config::block::BlockBuilder;
use crate::vmm_config::boot_source::DEFAULT_KERNEL_CMDLINE;
#[cfg(not(feature = "tee"))]
use crate::vmm_config::fs::FsBuilder;
use crate::vmm_config::fs::FsDeviceConfig;
#[cfg(target_os = "linux")]
use crate::vstate::KvmContext;
#[cfg(all(target_os = "linux", feature = "tee"))]
Expand Down Expand Up @@ -842,14 +842,17 @@ fn create_guest_memory(

if let Some(vm_resources) = vm_resources {
#[cfg(not(feature = "tee"))]
for (index, _fs) in vm_resources.fs.list.iter().enumerate() {
shm_manager
.create_fs_region(index, 1 << 29)
.map_err(StartMicrovmError::ShmCreate)?;
for (index, fs) in vm_resources.fs.iter().enumerate() {
if let Some(shm_size) = fs.shm_size {
shm_manager
.create_fs_region(index, shm_size)
.map_err(StartMicrovmError::ShmCreate)?;
}
}
if vm_resources.gpu_virgl_flags.is_some() {
let size = vm_resources.gpu_shm_size.unwrap_or(1 << 33);
shm_manager
.create_gpu_region(1 << 33)
.create_gpu_region(size)
.map_err(StartMicrovmError::ShmCreate)?;
}

Expand Down Expand Up @@ -1177,14 +1180,18 @@ fn attach_mmio_device(
#[cfg(not(feature = "tee"))]
fn attach_fs_devices(
vmm: &mut Vmm,
fs_devs: &FsBuilder,
fs_devs: &[FsDeviceConfig],
shm_manager: &mut ShmManager,
intc: Option<Arc<Mutex<Gic>>>,
#[cfg(target_os = "macos")] map_sender: Sender<MemoryMapping>,
) -> std::result::Result<(), StartMicrovmError> {
use self::StartMicrovmError::*;

for (i, fs) in fs_devs.list.iter().enumerate() {
for (i, config) in fs_devs.iter().enumerate() {
let fs = Arc::new(Mutex::new(
devices::virtio::Fs::new(config.fs_id.clone(), config.shared_dir.clone()).unwrap(),
));

let id = format!("{}{}", String::from(fs.lock().unwrap().id()), i);

if let Some(ref intc) = intc {
Expand Down
15 changes: 9 additions & 6 deletions src/vmm/src/resources.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,6 @@ pub enum Error {
/// Error opening TEE config file.
#[cfg(feature = "tee")]
OpenTeeConfig(std::io::Error),
/// Fs device configuration error.
#[cfg(not(feature = "tee"))]
FsDevice(FsConfigError),
/// Error parsing TEE config file.
#[cfg(feature = "tee")]
ParseTeeConfig(serde_json::Error),
Expand Down Expand Up @@ -96,7 +93,7 @@ pub struct VmResources {
pub initrd_bundle: Option<InitrdBundle>,
/// The fs device.
#[cfg(not(feature = "tee"))]
pub fs: FsBuilder,
pub fs: Vec<FsDeviceConfig>,
/// The vsock device.
pub vsock: VsockBuilder,
/// The virtio-blk device.
Expand All @@ -110,6 +107,7 @@ pub struct VmResources {
pub tee_config: TeeConfig,
/// Flags for the virtio-gpu device.
pub gpu_virgl_flags: Option<u32>,
pub gpu_shm_size: Option<usize>,
#[cfg(feature = "snd")]
/// Enable the virtio-snd device.
pub snd_device: bool,
Expand Down Expand Up @@ -236,8 +234,8 @@ impl VmResources {
}

#[cfg(not(feature = "tee"))]
pub fn add_fs_device(&mut self, config: FsDeviceConfig) -> Result<FsConfigError> {
self.fs.insert(config)
pub fn add_fs_device(&mut self, config: FsDeviceConfig) {
self.fs.push(config)
}

#[cfg(feature = "blk")]
Expand All @@ -254,6 +252,10 @@ impl VmResources {
self.gpu_virgl_flags = Some(virgl_flags);
}

pub fn set_gpu_shm_size(&mut self, shm_size: usize) {
self.gpu_shm_size = Some(shm_size);
}

#[cfg(feature = "snd")]
pub fn set_snd_device(&mut self, enabled: bool) {
self.snd_device = enabled;
Expand Down Expand Up @@ -326,6 +328,7 @@ mod tests {
#[cfg(feature = "net")]
net_builder: Default::default(),
gpu_virgl_flags: None,
gpu_shm_size: None,
#[cfg(feature = "snd")]
enable_snd: False,
console_output: None,
Expand Down
Loading

0 comments on commit 7b13b39

Please sign in to comment.