From fd493001f6eaf02123e42d5bb1e89315b36a3434 Mon Sep 17 00:00:00 2001 From: Priyansh Rathi Date: Sat, 19 Aug 2023 01:08:40 -0700 Subject: [PATCH] vsock: Add new VMs at runtime Implement ability to add new VMs at runtime. Can also modify the configuration for existing VMs but the modifications will take effect only after the VM is restarted. Removing a VM at runtime is not supported at the moment. To use this feature, you must use the `--config` CLI argument to specify the initial configuration with the `--watch` flag set. The path to the config file provided must be a symlink to the actual YAML config. To update the VM configuration at runtime, simply change the symlink to point to the YAML config file with the new configuration. Signed-off-by: Priyansh Rathi --- Cargo.lock | 118 +++++++++++++++++++ crates/vsock/Cargo.toml | 1 + crates/vsock/src/main.rs | 243 ++++++++++++++++++++++++++++----------- 3 files changed, 297 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9006c533..7db3893b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -100,6 +115,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -475,6 +505,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "glob" version = "0.3.1" @@ -533,6 +569,28 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "futures-core", + "inotify-sys", + "libc", + "tokio", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "intmap" version = "2.0.0" @@ -648,6 +706,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + [[package]] name = "nom" version = "7.1.3" @@ -689,6 +767,15 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -902,6 +989,12 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -997,6 +1090,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1100,6 +1203,20 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + [[package]] name = "toml" version = "0.5.11" @@ -1274,6 +1391,7 @@ dependencies = [ "env_logger", "epoll", "futures", + "inotify", "log", "serde", "serde_yaml", diff --git a/crates/vsock/Cargo.toml b/crates/vsock/Cargo.toml index 2b37c5613..bcacc5170 100644 --- a/crates/vsock/Cargo.toml +++ b/crates/vsock/Cargo.toml @@ -30,6 +30,7 @@ vmm-sys-util = "0.11" config = "0.13" serde = "1" serde_yaml = "0.9" +inotify = "0.10.2" [dev-dependencies] virtio-queue = { version = "0.9", features = ["test-utils"] } diff --git a/crates/vsock/src/main.rs b/crates/vsock/src/main.rs index fa38bfb64..03d059ac8 100644 --- a/crates/vsock/src/main.rs +++ b/crates/vsock/src/main.rs @@ -9,15 +9,17 @@ mod vhu_vsock_thread; mod vsock_conn; use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, convert::TryFrom, + path::Path, process::exit, - sync::{Arc, RwLock}, + sync::{Arc, Mutex, RwLock}, thread, }; use crate::vhu_vsock::{CidMap, VhostUserVsockBackend, VsockConfig}; use clap::{Args, Parser}; +use inotify::{EventMask, Inotify, WatchMask}; use log::{error, info, warn}; use serde::Deserialize; use thiserror::Error as ThisError; @@ -25,6 +27,9 @@ use vhost::{vhost_user, vhost_user::Listener}; use vhost_user_backend::VhostUserDaemon; use vm_memory::{GuestMemoryAtomic, GuestMemoryMmap}; +type ConfigMap = HashMap>>; +type JoinHandle = thread::JoinHandle>; + const DEFAULT_GUEST_CID: u64 = 3; const DEFAULT_TX_BUFFER_SIZE: u32 = 64 * 1024; const DEFAULT_GROUP_NAME: &str = "default"; @@ -101,6 +106,12 @@ struct ConfigFileVsockParam { groups: Option, } +#[derive(Debug, Clone)] +struct CliVsockConfig { + vsock_configs: Vec, + watched_config: Option, +} + #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct VsockArgs { @@ -118,6 +129,11 @@ struct VsockArgs { /// Load from a given configuration file #[arg(long)] config: Option, + + /// Watch the configuration file for changes and update the configuration accordingly at runtime. + /// This option is only valid when used with the `config` option. + #[arg(long, requires = "config")] + watch: bool, } fn parse_vm_params(s: &str) -> Result { @@ -155,58 +171,74 @@ fn parse_vm_params(s: &str) -> Result { )) } -impl VsockArgs { - pub fn parse_config(&self) -> Option, CliError>> { - if let Some(c) = &self.config { - let b = config::Config::builder() - .add_source(config::File::new(c.as_str(), config::FileFormat::Yaml)) - .build(); - if let Ok(s) = b { - let mut v = s.get::>("vms").unwrap(); - if !v.is_empty() { - let parsed: Vec = v - .drain(..) - .map(|p| { - VsockConfig::new( - p.guest_cid.unwrap_or(DEFAULT_GUEST_CID), - p.socket.trim().to_string(), - p.uds_path.trim().to_string(), - p.tx_buffer_size.unwrap_or(DEFAULT_TX_BUFFER_SIZE), - p.groups.map_or(vec![DEFAULT_GROUP_NAME.to_string()], |g| { - g.trim().split('+').map(String::from).collect() - }), - ) - }) - .collect(); - return Some(Ok(parsed)); - } else { - return Some(Err(CliError::ConfigParse)); - } - } else { - return Some(Err(CliError::ConfigParse)); - } +pub(crate) fn parse_config(config: &str) -> Result, CliError> { + let b = config::Config::builder() + .add_source(config::File::new(config, config::FileFormat::Yaml)) + .build(); + if let Ok(s) = b { + let mut v = s.get::>("vms").unwrap(); + if !v.is_empty() { + let parsed: Vec = v + .drain(..) + .map(|p| { + VsockConfig::new( + p.guest_cid.unwrap_or(DEFAULT_GUEST_CID), + p.socket.trim().to_string(), + p.uds_path.trim().to_string(), + p.tx_buffer_size.unwrap_or(DEFAULT_TX_BUFFER_SIZE), + p.groups.map_or(vec![DEFAULT_GROUP_NAME.to_string()], |g| { + g.trim().split('+').map(String::from).collect() + }), + ) + }) + .collect(); + Ok(parsed) + } else { + Err(CliError::ConfigParse) } - None + } else { + Err(CliError::ConfigParse) } } -impl TryFrom for Vec { +impl CliVsockConfig { + pub fn new(vsock_configs: Vec, watched_config: Option) -> Self { + Self { + vsock_configs, + watched_config, + } + } +} + +impl TryFrom for CliVsockConfig { type Error = CliError; fn try_from(cmd_args: VsockArgs) -> Result { // we try to use the configuration first, if failed, then fall back to the manual settings. - match cmd_args.parse_config() { - Some(c) => c, + match &cmd_args.config { + Some(c) => parse_config(c).map(|v| { + Self::new( + v, + if cmd_args.watch { + cmd_args.config + } else { + None + }, + ) + }), _ => match cmd_args.vm { - Some(v) => Ok(v), + Some(v) => Ok(Self::new(v, None)), _ => cmd_args.param.map_or(Err(CliError::NoArgsProvided), |p| { - Ok(vec![VsockConfig::new( - p.guest_cid, - p.socket.trim().to_string(), - p.uds_path.trim().to_string(), - p.tx_buffer_size, - p.groups.trim().split('+').map(String::from).collect(), - )]) + Ok(CliVsockConfig::new( + vec![VsockConfig::new( + p.guest_cid, + p.socket.trim().to_string(), + p.uds_path.trim().to_string(), + p.tx_buffer_size, + p.groups.trim().split('+').map(String::from).collect(), + )], + None, + )) }), }, } @@ -215,11 +247,13 @@ impl TryFrom for Vec { /// This is the public API through which an external program starts the /// vhost-device-vsock backend server. -pub(crate) fn start_backend_server( - config: VsockConfig, +pub(crate) fn start_backend_server_thread( + config: Arc>, cid_map: Arc>, ) -> Result<(), BackendError> { loop { + let config = config.read().unwrap().clone(); + let backend = Arc::new( VhostUserVsockBackend::new(config.clone(), cid_map.clone()) .map_err(BackendError::CouldNotCreateBackend)?, @@ -264,21 +298,98 @@ pub(crate) fn start_backend_server( } } -pub(crate) fn start_backend_servers(configs: &[VsockConfig]) -> Result<(), BackendError> { +pub(crate) fn start_backend_server( + config: &VsockConfig, + config_map: &mut ConfigMap, + cid_map: Arc>, + handles: Arc>>, +) { + let c = Arc::new(RwLock::new(config.clone())); + let guest_cid = config.get_guest_cid(); + config_map.insert(guest_cid, c.clone()); + + let cid_map = cid_map.clone(); + + let handle = thread::Builder::new() + .name(format!("vhu-vsock-cid-{}", guest_cid)) + .spawn(move || start_backend_server_thread(c, cid_map)) + .unwrap(); + handles.lock().unwrap().push_back(handle); +} + +pub(crate) fn start_config_watcher( + config_map: ConfigMap, + cid_map: Arc>, + handles: Arc>>, + watched_config: String, +) -> Result<(), BackendError> { + let mut config_map = config_map; + + let mut inotify = Inotify::init().unwrap(); // could be wrapped as a BackendError + + let watched_config_path = Path::new(&watched_config); + let watched_config_file_name = watched_config_path.file_name().unwrap(); + let watched_config_dir = watched_config_path.parent().unwrap(); + + inotify + .watches() + .add(watched_config_dir, WatchMask::MOVED_TO) + .unwrap(); // could be wrapped as a BackendError + + let mut buffer = [0u8; 4096]; + loop { + let events = inotify.read_events_blocking(&mut buffer).unwrap(); + + for event in events { + if event.mask.contains(EventMask::MOVED_TO) { + let dest_file_name = event.name.unwrap(); + if dest_file_name == watched_config_file_name { + let vsock_configs = parse_config(&watched_config).unwrap(); // could be wrapped as a BackendError + for c in vsock_configs.iter() { + let guest_cid = c.get_guest_cid(); + if config_map.contains_key(&guest_cid) { + let config = config_map.get(&guest_cid).unwrap(); + let mut config = config.write().unwrap(); + *config = c.clone(); + } else { + start_backend_server( + c, + &mut config_map, + cid_map.clone(), + handles.clone(), + ); + } + } + } + } + } + } +} + +pub(crate) fn start_backend_servers(cli_vsock_config: &CliVsockConfig) -> Result<(), BackendError> { + let mut config_map: ConfigMap = HashMap::new(); let cid_map: Arc> = Arc::new(RwLock::new(HashMap::new())); - let mut handles = Vec::new(); + let handles = Arc::new(Mutex::new(VecDeque::new())); - for c in configs.iter() { - let config = c.clone(); - let cid_map = cid_map.clone(); + for c in cli_vsock_config.vsock_configs.iter() { + start_backend_server(c, &mut config_map, cid_map.clone(), handles.clone()); + } + + if let Some(watched_config) = cli_vsock_config.watched_config.clone() { + let handles2 = handles.clone(); let handle = thread::Builder::new() - .name(format!("vhu-vsock-cid-{}", c.get_guest_cid())) - .spawn(move || start_backend_server(config, cid_map)) + .name("vhu-vsock-config-watcher".to_string()) + .spawn(move || start_config_watcher(config_map, cid_map, handles2, watched_config)) .unwrap(); - handles.push(handle); + handles.lock().unwrap().push_back(handle); } - for handle in handles { + loop { + if handles.lock().unwrap().is_empty() { + break; + } + + let handle = handles.lock().unwrap().pop_front().unwrap(); handle.join().unwrap()?; } @@ -288,7 +399,7 @@ pub(crate) fn start_backend_servers(configs: &[VsockConfig]) -> Result<(), Backe fn main() { env_logger::init(); - let configs = match Vec::::try_from(VsockArgs::parse()) { + let cli_vsock_config = match CliVsockConfig::try_from(VsockArgs::parse()) { Ok(c) => c, Err(e) => { println!("Error parsing arguments: {}", e); @@ -296,7 +407,7 @@ fn main() { } }; - if let Err(e) = start_backend_servers(&configs) { + if let Err(e) = start_backend_servers(&cli_vsock_config) { error!("{e}"); exit(1); } @@ -327,6 +438,7 @@ mod tests { }), vm: None, config: None, + watch: false, } } fn from_file(config: &str) -> Self { @@ -334,6 +446,7 @@ mod tests { param: None, vm: None, config: Some(config.to_string()), + watch: false, } } } @@ -346,10 +459,10 @@ mod tests { let uds_path = test_dir.path().join("vm4.vsock").display().to_string(); let args = VsockArgs::from_args(3, &socket_path, &uds_path, 64 * 1024, "group1"); - let configs = Vec::::try_from(args); - assert!(configs.is_ok()); + let cli_vsock_config = CliVsockConfig::try_from(args); + assert!(cli_vsock_config.is_ok()); - let configs = configs.unwrap(); + let configs = cli_vsock_config.unwrap().vsock_configs; assert_eq!(configs.len(), 1); let config = &configs[0]; @@ -393,10 +506,10 @@ mod tests { let args = VsockArgs::parse_from(params); - let configs = Vec::::try_from(args); - assert!(configs.is_ok()); + let cli_vsock_config = CliVsockConfig::try_from(args); + assert!(cli_vsock_config.is_ok()); - let configs = configs.unwrap(); + let configs = cli_vsock_config.unwrap().vsock_configs; assert_eq!(configs.len(), 3); let config = configs.get(0).unwrap(); @@ -460,7 +573,7 @@ mod tests { .unwrap(); let args = VsockArgs::from_file(&config_path.display().to_string()); - let configs = Vec::::try_from(args).unwrap(); + let configs = CliVsockConfig::try_from(args).unwrap().vsock_configs; assert_eq!(configs.len(), 1); let config = &configs[0]; @@ -488,7 +601,7 @@ mod tests { .unwrap(); let args = VsockArgs::from_file(&config_path.display().to_string()); - let configs = Vec::::try_from(args).unwrap(); + let configs = CliVsockConfig::try_from(args).unwrap().vsock_configs; assert_eq!(configs.len(), 1); let config = &configs[0];