Skip to content

Commit 91277a7

Browse files
syntacticallysyntactically
committed
[hyperlight_host] Plumb a trace file descriptor around
This adds (unused) support for creating trace files for sandboxes and passing them around to relevant sandbox event handler code. This will be used for collecting debug trace and profiling information. Signed-off-by: Lucy Menon <[email protected]>
1 parent 3574bca commit 91277a7

File tree

10 files changed

+170
-19
lines changed

10 files changed

+170
-19
lines changed

‎src/hyperlight_host/Cargo.toml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ strum = { version = "0.26", features = ["derive"] }
4848
tempfile = { version = "3.10", optional = true }
4949
serde_yaml = "0.9"
5050
anyhow = "1.0"
51+
uuid = { version = "1.4.1", features = ["v4"] }
52+
5153

5254
[target.'cfg(windows)'.dependencies]
5355
windows = { version = "0.58", features = [
@@ -76,7 +78,6 @@ kvm-bindings = { version = "0.10.0", features = ["fam-wrappers"], optional = tru
7678
kvm-ioctls = { version = "0.19.0", optional = true }
7779

7880
[dev-dependencies]
79-
uuid = { version = "1.4.1", features = ["v4"] }
8081
signal-hook-registry = "1.4.1"
8182
envy = { version = "0.4.2" }
8283
serde = "1.0"
@@ -120,6 +121,10 @@ executable_heap = []
120121
# This feature enables printing of debug information to stdout in debug builds
121122
print_debug = []
122123
crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
124+
# This feature enables the generation of trace files. Traces do not
125+
# include any particularly expensive instrumentation unless other
126+
# features are enabled.
127+
trace_guest = []
123128
kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
124129
mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
125130
inprocess = []

‎src/hyperlight_host/src/hypervisor/handlers.rs

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,24 @@ use std::sync::{Arc, Mutex};
1818

1919
use tracing::{instrument, Span};
2020

21+
#[cfg(feature = "trace_guest")]
22+
use super::Hypervisor;
23+
#[cfg(feature = "trace_guest")]
24+
use crate::sandbox::TraceInfo;
2125
use crate::{new_error, Result};
2226

2327
/// The trait representing custom logic to handle the case when
2428
/// a Hypervisor's virtual CPU (vCPU) informs Hyperlight the guest
2529
/// has initiated an outb operation.
2630
pub(crate) trait OutBHandlerCaller: Sync + Send {
2731
/// Function that gets called when an outb operation has occurred.
28-
fn call(&mut self, port: u16, payload: u64) -> Result<()>;
32+
fn call(
33+
&mut self,
34+
#[cfg(feature = "trace_guest")] hv: &mut dyn Hypervisor,
35+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
36+
port: u16,
37+
payload: u64,
38+
) -> Result<()>;
2939
}
3040

3141
/// A convenient type representing a common way `OutBHandler` implementations
@@ -36,6 +46,10 @@ pub(crate) trait OutBHandlerCaller: Sync + Send {
3646
/// a &mut self).
3747
pub(crate) type OutBHandlerWrapper = Arc<Mutex<dyn OutBHandlerCaller>>;
3848

49+
#[cfg(feature = "trace_guest")]
50+
pub(crate) type OutBHandlerFunction =
51+
Box<dyn FnMut(&mut dyn Hypervisor, TraceInfo, u16, u64) -> Result<()> + Send>;
52+
#[cfg(not(feature = "trace_guest"))]
3953
pub(crate) type OutBHandlerFunction = Box<dyn FnMut(u16, u64) -> Result<()> + Send>;
4054

4155
/// A `OutBHandler` implementation using a `OutBHandlerFunction`
@@ -52,12 +66,25 @@ impl From<OutBHandlerFunction> for OutBHandler {
5266

5367
impl OutBHandlerCaller for OutBHandler {
5468
#[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
55-
fn call(&mut self, port: u16, payload: u64) -> Result<()> {
69+
fn call(
70+
&mut self,
71+
#[cfg(feature = "trace_guest")] hv: &mut dyn Hypervisor,
72+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
73+
port: u16,
74+
payload: u64,
75+
) -> Result<()> {
5676
let mut func = self
5777
.0
5878
.try_lock()
5979
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?;
60-
func(port, payload)
80+
func(
81+
#[cfg(feature = "trace_guest")]
82+
hv,
83+
#[cfg(feature = "trace_guest")]
84+
trace_info,
85+
port,
86+
payload,
87+
)
6188
}
6289
}
6390

‎src/hyperlight_host/src/hypervisor/hyperv_linux.rs

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ use crate::hypervisor::hypervisor_handler::HypervisorHandler;
3737
use crate::hypervisor::HyperlightExit;
3838
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
3939
use crate::mem::ptr::{GuestPtr, RawPtr};
40+
#[cfg(feature = "trace_guest")]
41+
use crate::sandbox::TraceInfo;
4042
use crate::{log_then_return, new_error, Result};
4143

4244
/// Determine whether the HyperV for Linux hypervisor API is present
@@ -173,6 +175,7 @@ impl Hypervisor for HypervLinuxDriver {
173175
outb_hdl: OutBHandlerWrapper,
174176
mem_access_hdl: MemAccessHandlerWrapper,
175177
hv_handler: Option<HypervisorHandler>,
178+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
176179
) -> Result<()> {
177180
let regs = StandardRegisters {
178181
rip: self.entrypoint,
@@ -194,6 +197,8 @@ impl Hypervisor for HypervLinuxDriver {
194197
hv_handler,
195198
outb_hdl,
196199
mem_access_hdl,
200+
#[cfg(feature = "trace_guest")]
201+
trace_info,
197202
)?;
198203

199204
// reset RSP to what it was before initialise
@@ -212,6 +217,7 @@ impl Hypervisor for HypervLinuxDriver {
212217
outb_handle_fn: OutBHandlerWrapper,
213218
mem_access_fn: MemAccessHandlerWrapper,
214219
hv_handler: Option<HypervisorHandler>,
220+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
215221
) -> Result<()> {
216222
// Reset general purpose registers except RSP, then set RIP
217223
let rsp_before = self.vcpu_fd.get_regs()?.rsp;
@@ -238,6 +244,8 @@ impl Hypervisor for HypervLinuxDriver {
238244
hv_handler,
239245
outb_handle_fn,
240246
mem_access_fn,
247+
#[cfg(feature = "trace_guest")]
248+
trace_info,
241249
)?;
242250

243251
// reset RSP to what it was before function call
@@ -257,12 +265,20 @@ impl Hypervisor for HypervLinuxDriver {
257265
rip: u64,
258266
instruction_length: u64,
259267
outb_handle_fn: OutBHandlerWrapper,
268+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
260269
) -> Result<()> {
261270
let payload = data[..8].try_into()?;
262271
outb_handle_fn
263272
.try_lock()
264273
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
265-
.call(port, u64::from_le_bytes(payload))?;
274+
.call(
275+
#[cfg(feature = "trace_guest")]
276+
self,
277+
#[cfg(feature = "trace_guest")]
278+
trace_info,
279+
port,
280+
u64::from_le_bytes(payload),
281+
)?;
266282

267283
// update rip
268284
self.vcpu_fd.set_reg(&[hv_register_assoc {

‎src/hyperlight_host/src/hypervisor/hyperv_windows.rs

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ use crate::hypervisor::wrappers::WHvGeneralRegisters;
4545
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
4646
use crate::mem::ptr::{GuestPtr, RawPtr};
4747
use crate::HyperlightError::{NoHypervisorFound, WindowsAPIError};
48+
#[cfg(feature = "trace_guest")]
49+
use crate::sandbox::TraceInfo;
4850
use crate::{debug, log_then_return, new_error, Result};
4951

5052
/// A Hypervisor driver for HyperV-on-Windows.
@@ -311,6 +313,7 @@ impl Hypervisor for HypervWindowsDriver {
311313
outb_hdl: OutBHandlerWrapper,
312314
mem_access_hdl: MemAccessHandlerWrapper,
313315
hv_handler: Option<HypervisorHandler>,
316+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
314317
) -> Result<()> {
315318
let regs = WHvGeneralRegisters {
316319
rip: self.entrypoint,
@@ -332,6 +335,8 @@ impl Hypervisor for HypervWindowsDriver {
332335
hv_handler,
333336
outb_hdl,
334337
mem_access_hdl,
338+
#[cfg(feature = "trace_guest")]
339+
trace_info,
335340
)?;
336341

337342
// reset RSP to what it was before initialise
@@ -350,6 +355,7 @@ impl Hypervisor for HypervWindowsDriver {
350355
outb_hdl: OutBHandlerWrapper,
351356
mem_access_hdl: MemAccessHandlerWrapper,
352357
hv_handler: Option<HypervisorHandler>,
358+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
353359
) -> Result<()> {
354360
// Reset general purpose registers except RSP, then set RIP
355361
let rsp_before = self.processor.get_regs()?.rsp;
@@ -374,6 +380,8 @@ impl Hypervisor for HypervWindowsDriver {
374380
hv_handler,
375381
outb_hdl,
376382
mem_access_hdl,
383+
#[cfg(feature = "trace_guest")]
384+
trace_info,
377385
)?;
378386

379387
// reset RSP to what it was before function call
@@ -393,12 +401,20 @@ impl Hypervisor for HypervWindowsDriver {
393401
rip: u64,
394402
instruction_length: u64,
395403
outb_handle_fn: OutBHandlerWrapper,
404+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
396405
) -> Result<()> {
397406
let payload = data[..8].try_into()?;
398407
outb_handle_fn
399408
.try_lock()
400409
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
401-
.call(port, u64::from_le_bytes(payload))?;
410+
.call(
411+
#[cfg(feature = "trace_guest")]
412+
self,
413+
#[cfg(feature = "trace_guest")]
414+
trace_info,
415+
port,
416+
u64::from_le_bytes(payload),
417+
)?;
402418

403419
let mut regs = self.processor.get_regs()?;
404420
regs.rip = rip + instruction_length;

‎src/hyperlight_host/src/hypervisor/hypervisor_handler.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory, SharedMemory};
4747
use crate::sandbox::hypervisor::{get_available_hypervisor, HypervisorType};
4848
#[cfg(feature = "function_call_metrics")]
4949
use crate::sandbox::metrics::SandboxMetric::GuestFunctionCallDurationMicroseconds;
50+
#[cfg(feature = "trace_guest")]
51+
use crate::sandbox::TraceInfo;
5052
#[cfg(target_os = "linux")]
5153
use crate::signal_handlers::setup_signal_handlers;
5254
use crate::HyperlightError::{
@@ -183,6 +185,8 @@ pub(crate) struct HvHandlerConfig {
183185
pub(crate) outb_handler: OutBHandlerWrapper,
184186
pub(crate) mem_access_handler: MemAccessHandlerWrapper,
185187
pub(crate) max_wait_for_cancellation: Duration,
188+
#[cfg(feature = "trace_guest")]
189+
pub(crate) trace_info: TraceInfo,
186190
}
187191

188192
impl HypervisorHandler {
@@ -338,6 +342,8 @@ impl HypervisorHandler {
338342
configuration.outb_handler.clone(),
339343
configuration.mem_access_handler.clone(),
340344
Some(hv_handler_clone.clone()),
345+
#[cfg(feature = "trace_guest")]
346+
configuration.trace_info.clone(),
341347
);
342348
drop(mem_lock_guard);
343349
drop(evar_lock_guard);
@@ -417,6 +423,7 @@ impl HypervisorHandler {
417423
configuration.outb_handler.clone(),
418424
configuration.mem_access_handler.clone(),
419425
Some(hv_handler_clone.clone()),
426+
#[cfg(feature = "trace_guest")] configuration.trace_info.clone(),
420427
);
421428
histogram_vec_observe!(
422429
&GuestFunctionCallDurationMicroseconds,
@@ -432,6 +439,7 @@ impl HypervisorHandler {
432439
configuration.outb_handler.clone(),
433440
configuration.mem_access_handler.clone(),
434441
Some(hv_handler_clone.clone()),
442+
#[cfg(feature = "trace_guest")] configuration.trace_info.clone(),
435443
)
436444
};
437445
drop(mem_lock_guard);

‎src/hyperlight_host/src/hypervisor/kvm.rs

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ use super::{
3131
use crate::hypervisor::hypervisor_handler::HypervisorHandler;
3232
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
3333
use crate::mem::ptr::{GuestPtr, RawPtr};
34+
#[cfg(feature = "trace_guest")]
35+
use crate::sandbox::TraceInfo;
3436
use crate::{log_then_return, new_error, Result};
3537

3638
/// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise
@@ -167,6 +169,7 @@ impl Hypervisor for KVMDriver {
167169
outb_hdl: OutBHandlerWrapper,
168170
mem_access_hdl: MemAccessHandlerWrapper,
169171
hv_handler: Option<HypervisorHandler>,
172+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
170173
) -> Result<()> {
171174
let regs = kvm_regs {
172175
rip: self.entrypoint,
@@ -187,6 +190,8 @@ impl Hypervisor for KVMDriver {
187190
hv_handler,
188191
outb_hdl,
189192
mem_access_hdl,
193+
#[cfg(feature = "trace_guest")]
194+
trace_info,
190195
)?;
191196

192197
// reset RSP to what it was before initialise
@@ -204,6 +209,7 @@ impl Hypervisor for KVMDriver {
204209
outb_handle_fn: OutBHandlerWrapper,
205210
mem_access_fn: MemAccessHandlerWrapper,
206211
hv_handler: Option<HypervisorHandler>,
212+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
207213
) -> Result<()> {
208214
// Reset general purpose registers except RSP, then set RIP
209215
let rsp_before = self.vcpu_fd.get_regs()?.rsp;
@@ -229,6 +235,8 @@ impl Hypervisor for KVMDriver {
229235
hv_handler,
230236
outb_handle_fn,
231237
mem_access_fn,
238+
#[cfg(feature = "trace_guest")]
239+
trace_info,
232240
)?;
233241

234242
// reset RSP to what it was before function call
@@ -247,6 +255,7 @@ impl Hypervisor for KVMDriver {
247255
_rip: u64,
248256
_instruction_length: u64,
249257
outb_handle_fn: OutBHandlerWrapper,
258+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
250259
) -> Result<()> {
251260
// KVM does not need RIP or instruction length, as it automatically sets the RIP
252261

@@ -260,7 +269,14 @@ impl Hypervisor for KVMDriver {
260269
outb_handle_fn
261270
.try_lock()
262271
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
263-
.call(port, payload_u64)?;
272+
.call(
273+
#[cfg(feature = "trace_guest")]
274+
self,
275+
#[cfg(feature = "trace_guest")]
276+
trace_info,
277+
port,
278+
payload_u64,
279+
)?;
264280
}
265281

266282
Ok(())
@@ -388,16 +404,21 @@ pub(crate) mod test_cfg {
388404
mod tests {
389405
use std::sync::{Arc, Mutex};
390406

391-
use crate::hypervisor::handlers::{MemAccessHandler, OutBHandler};
407+
use crate::hypervisor::handlers::{MemAccessHandler, OutBHandler, OutBHandlerFunction};
392408
use crate::hypervisor::tests::test_initialise;
393409
use crate::{should_run_kvm_linux_test, Result};
394410

395411
#[test]
396412
fn test_init() {
397413
should_run_kvm_linux_test!();
398414
let outb_handler = {
399-
let func: Box<dyn FnMut(u16, u64) -> Result<()> + Send> =
400-
Box::new(|_, _| -> Result<()> { Ok(()) });
415+
let func: OutBHandlerFunction = Box::new(
416+
|#[cfg(feature = "trace_guest")] _,
417+
#[cfg(feature = "trace_guest")] _,
418+
_,
419+
_|
420+
-> Result<()> { Ok(()) },
421+
);
401422
Arc::new(Mutex::new(OutBHandler::from(func)))
402423
};
403424
let mem_access_handler = {

0 commit comments

Comments
 (0)