diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 8721f445..2350756b 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -135,6 +135,12 @@ jobs: RUST_LOG: debug run: just test-rust-gdb-debugging ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + - name: Run Rust Crashdump tests + env: + CARGO_TERM_COLOR: always + RUST_LOG: debug + run: just test-rust-crashdump ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + ### Benchmarks ### - name: Install github-cli (Linux mariner) if: runner.os == 'Linux' && matrix.hypervisor == 'mshv' diff --git a/Cargo.lock b/Cargo.lock index 03992af3..4efe4a2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,6 +663,19 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elfcore" +version = "1.1.5" +source = "git+https://github.com/hyperlight-dev/elfcore.git?rev=cef4c80e26bf4b2a5599e50d2d1730965f942c13#cef4c80e26bf4b2a5599e50d2d1730965f942c13" +dependencies = [ + "libc", + "nix", + "smallvec", + "thiserror 1.0.69", + "tracing", + "zerocopy 0.7.35", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1188,10 +1201,12 @@ dependencies = [ "built", "cfg-if", "cfg_aliases", + "chrono", "criterion", "crossbeam", "crossbeam-channel", "crossbeam-queue", + "elfcore", "env_logger", "flatbuffers", "gdbstub", @@ -1874,6 +1889,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" diff --git a/Justfile b/Justfile index 634341f0..42dedbfd 100644 --- a/Justfile +++ b/Justfile @@ -72,6 +72,9 @@ test-like-ci config=default-target hypervisor="kvm": @# without any driver (should fail to compile) just test-compilation-fail {{config}} + @# test the crashdump feature + just test-rust-crashdump {{config}} + # runs all tests test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration "rust" target features) (test-integration "c" target features) (test-seccomp target features) @@ -116,6 +119,10 @@ test-rust-gdb-debugging target=default-target features="": cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- test_gdb +# rust test for crashdump +test-rust-crashdump target=default-target features="": + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }} -- test_crashdump + ################ ### LINTING #### diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index abf57106..48f54123 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -202,3 +202,222 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or └─┘ │ | | | │ | └───────────────────────────────────────────────────────────────────────────────────────────────┘ ``` + +## Dumping the guest state to an ELF core dump when an unhandled crash occurs + +When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state is dumped to an `ELF` core dump file. +This can be used to inspect the state of the guest at the time of the crash. + +To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, set the feature `crashdump` and run a debug build. +This will result in a dump file being created in the temporary directory. +The name and location of the dump file will be printed to the console and logged as an error message. + +### Inspecting the core dump + +After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`. +**NOTE: This feature has been tested with version `15.0` of `gdb` and version `17` of `lldb`, earlier versions may not work, it is recommended to use these versions or later.** + +To do this in vscode, the following configuration can be used to add debug configurations: + +```vscode +{ + "version": "0.2.0", + "inputs": [ + { + "id": "core_dump", + "type": "promptString", + "description": "Path to the core dump file", + }, + { + "id": "program", + "type": "promptString", + "description": "Path to the program to debug", + } + ], + "configurations": [ + { + "name": "[GDB] Load core dump file", + "type": "cppdbg", + "request": "launch", + "program": "${input:program}", + "coreDumpPath": "${input:core_dump}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + "externalConsole": false, + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + }, + { + "name": "[LLDB] Load core dump file", + "type": "lldb", + "request": "launch", + "stopOnEntry": true, + "processCreateCommands": [], + "targetCreateCommands": [ + "target create -c ${input:core_dump} ${input:program}", + ], + }, + ] +} +``` +**NOTE: The `CodeLldb` debug session does not stop after launching. To see the code, stack frames and registers you need to +press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245). +The `cppdbg` extension works as expected and stops at the entry point of the program.** + +## Compiling guests with debug information for release builds + +This section explains how to compile a guest with debugging information but still have optimized code, and how to separate the debug information from the binary. + +### Creating a release build with debug information + +To create a release build with debug information, you can add a custom profile to your `Cargo.toml` file: + +```toml +[profile.release-with-debug] +inherits = "release" +debug = true +``` + +This creates a new profile called `release-with-debug` that inherits all settings from the release profile but adds debug information. + +### Splitting debug information from the binary + +To reduce the binary size while still having debug information available, you can split the debug information into a separate file. +This is useful for production environments where you want smaller binaries but still want to be able to debug crashes. + +Here's a step-by-step guide: + +1. Build your guest with the release-with-debug profile: + ```bash + cargo build --profile release-with-debug + ``` + +2. Locate your binary in the target directory: + ```bash + TARGET_DIR="target" + PROFILE="release-with-debug" + ARCH="x86_64-unknown-none" # Your target architecture + BUILD_DIR="${TARGET_DIR}/${ARCH}/${PROFILE}" + BINARY=$(find "${BUILD_DIR}" -type f -executable -name "guest-binary" | head -1) + ``` + +3. Extract debug information into a full debug file: + ```bash + DEBUG_FILE_FULL="${BINARY}.debug.full" + objcopy --only-keep-debug "${BINARY}" "${DEBUG_FILE_FULL}" + ``` + +4. Create a symbols-only debug file (smaller, but still useful for stack traces): + ```bash + DEBUG_FILE="${BINARY}.debug" + objcopy --keep-file-symbols "${DEBUG_FILE_FULL}" "${DEBUG_FILE}" + ``` + +5. Strip debug information from the original binary but keep function names: + ```bash + objcopy --strip-debug "${BINARY}" + ``` + +6. Add a debug link to the stripped binary: + ```bash + objcopy --add-gnu-debuglink="${DEBUG_FILE}" "${BINARY}" + ``` + +After these steps, you'll have: +- An optimized binary with function names for basic stack traces +- A symbols-only debug file for stack traces +- A full debug file for complete source-level debugging + +### Analyzing core dumps with the debug files + +When you have a core dump from a crashed guest, you can analyze it with different levels of detail using either GDB or LLDB. + +#### Using GDB + +1. For basic analysis with function names (stack traces): + ```bash + gdb ${BINARY} -c /path/to/core.dump + ``` + +2. For full source-level debugging: + ```bash + gdb -s ${DEBUG_FILE_FULL} ${BINARY} -c /path/to/core.dump + ``` + +#### Using LLDB + +LLDB provides similar capabilities with slightly different commands: + +1. For basic analysis with function names (stack traces): + ```bash + lldb ${BINARY} -c /path/to/core.dump + ``` + +2. For full source-level debugging: + ```bash + lldb -o "target create -c /path/to/core.dump ${BINARY}" -o "add-dsym ${DEBUG_FILE_FULL}" + ``` + +3. If your debug symbols are in a separate file: + ```bash + lldb ${BINARY} -c /path/to/core.dump + (lldb) add-dsym ${DEBUG_FILE_FULL} + ``` + +### VSCode Debug Configurations + +You can configure VSCode (in `.vscode/launch.json`) to use these files by modifying the debug configurations: + +#### For GDB + +```json +{ + "name": "[GDB] Load core dump with full debug symbols", + "type": "cppdbg", + "request": "launch", + "program": "${input:program}", + "coreDumpPath": "${input:core_dump}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + "externalConsole": false, + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] +} +``` + +#### For LLDB + +```json +{ + "name": "[LLDB] Load core dump with full debug symbols", + "type": "lldb", + "request": "launch", + "program": "${input:program}", + "cwd": "${workspaceFolder}", + "processCreateCommands": [], + "targetCreateCommands": [ + "target create -c ${input:core_dump} ${input:program}" + ], + "postRunCommands": [ + // if debug symbols are in a different file + "add-dsym ${input:debug_file_path}" + ] +} +``` diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 2ffb8772..1e7ad70c 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -38,9 +38,10 @@ vmm-sys-util = "0.13.0" crossbeam = "0.8.0" crossbeam-channel = "0.5.15" thiserror = "2.0.12" -tempfile = { version = "3.20", optional = true } +chrono = { version = "0.4", optional = true } anyhow = "1.0" metrics = "0.24.2" +elfcore = { git = "https://github.com/hyperlight-dev/elfcore.git", rev = "cef4c80e26bf4b2a5599e50d2d1730965f942c13" } [target.'cfg(windows)'.dependencies] windows = { version = "0.61", features = [ @@ -122,7 +123,8 @@ function_call_metrics = [] executable_heap = [] # This feature enables printing of debug information to stdout in debug builds print_debug = [] -crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. +# Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. +crashdump = ["dep:chrono"] kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"] mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"] diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 4710d3cb..779fd754 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -93,8 +93,7 @@ fn main() -> Result<()> { gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, - // crashdump feature is aliased with debug_assertions to make it only available in debug-builds. - crashdump: { all(feature = "crashdump", debug_assertions) }, + crashdump: { all(feature = "crashdump") }, // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. print_debug: { all(feature = "print_debug", debug_assertions) }, // the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index a70dc34c..b24ad3a5 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -1,44 +1,468 @@ +use std::cmp::min; use std::io::Write; -use tempfile::NamedTempFile; +use chrono; +use elfcore::{ + ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource, + ReadProcessMemory, ThreadView, VaProtection, VaRegion, +}; use super::Hypervisor; +use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::{new_error, Result}; -/// Dump registers + memory regions + raw memory to a tempfile -#[cfg(crashdump)] -pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { - let mut temp_file = NamedTempFile::with_prefix("mem")?; - let hv_details = format!("{:#x?}", hv); +/// This constant is used to identify the XSAVE state in the core dump +const NT_X86_XSTATE: u32 = 0x202; +/// This constant identifies the entry point of the program in an Auxiliary Vector +/// note of ELF. This tells a debugger whether the entry point of the program changed +/// so it can load the symbols correctly. +const AT_ENTRY: u64 = 9; +/// This constant is used to mark the end of the Auxiliary Vector note +const AT_NULL: u64 = 0; +/// The PID of the core dump process - this is a placeholder value +const CORE_DUMP_PID: i32 = 1; +/// The page size of the core dump +const CORE_DUMP_PAGE_SIZE: usize = 0x1000; - // write hypervisor details such as registers, info about mapped memory regions, etc. - temp_file.write_all(hv_details.as_bytes())?; - temp_file.write_all(b"================ MEMORY DUMP =================\n")?; +/// Structure to hold the crash dump context +/// This structure contains the information needed to create a core dump +#[derive(Debug)] +pub(crate) struct CrashDumpContext<'a> { + regions: &'a [MemoryRegion], + regs: [u64; 27], + xsave: Vec, + entry: u64, + binary: Option, + filename: Option, +} + +impl<'a> CrashDumpContext<'a> { + pub(crate) fn new( + regions: &'a [MemoryRegion], + regs: [u64; 27], + xsave: Vec, + entry: u64, + binary: Option, + filename: Option, + ) -> Self { + Self { + regions, + regs, + xsave, + entry, + binary, + filename, + } + } +} + +/// Structure that contains the process information for the core dump +/// This serves as a source of information for `elfcore`'s [`CoreDumpBuilder`] +struct GuestView { + regions: Vec, + threads: Vec, + aux_vector: Vec, +} + +impl GuestView { + fn new(ctx: &CrashDumpContext) -> Self { + // Map the regions to the format `CoreDumpBuilder` expects + let regions = ctx + .regions + .iter() + .filter(|r| !r.host_region.is_empty()) + .map(|r| VaRegion { + begin: r.guest_region.start as u64, + end: r.guest_region.end as u64, + offset: r.host_region.start as u64, + protection: VaProtection { + is_private: false, + read: r.flags.contains(MemoryRegionFlags::READ), + write: r.flags.contains(MemoryRegionFlags::WRITE), + execute: r.flags.contains(MemoryRegionFlags::EXECUTE), + }, + mapped_file_name: None, + }) + .collect(); + + let filename = ctx + .filename + .as_ref() + .map_or("".to_string(), |s| s.to_string()); + + let cmd = ctx + .binary + .as_ref() + .map_or("".to_string(), |s| s.to_string()); - // write the raw memory dump for each memory region - for region in hv.get_memory_regions() { - if region.host_region.start == 0 || region.host_region.is_empty() { - continue; + // The xsave state is checked as it can be empty + let mut components = vec![]; + if !ctx.xsave.is_empty() { + components.push(ArchComponentState { + name: "XSAVE", + note_type: NT_X86_XSTATE, + note_name: b"LINUX", + data: ctx.xsave.clone(), + }); } - // SAFETY: we got this memory region from the hypervisor so should never be invalid - let region_slice = unsafe { - std::slice::from_raw_parts( - region.host_region.start as *const u8, - region.host_region.len(), - ) + + // Create the thread view + // The thread view contains the information about the thread + // NOTE: Some of these fields are not used in the current implementation + let thread = ThreadView { + flags: 0, // Kernel flags for the process + tid: 1, + uid: 0, // User ID + gid: 0, // Group ID + comm: filename, + ppid: 0, // Parent PID + pgrp: 0, // Process group ID + nice: 0, // Nice value + state: 0, // Process state + utime: 0, // User time + stime: 0, // System time + cutime: 0, // Children User time + cstime: 0, // Children User time + cursig: 0, // Current signal + session: 0, // Session ID of the process + sighold: 0, // Blocked signal + sigpend: 0, // Pending signal + cmd_line: cmd, + + arch_state: Box::new(ArchState { + gpr_state: ctx.regs.to_vec(), + components, + }), }; - temp_file.write_all(region_slice)?; + + // Create the auxv vector + // The first entry is AT_ENTRY, which is the entry point of the program + // The entry point is the address where the program starts executing + // This helps the debugger to know that the entry is changed by an offset + // so the symbols can be loaded correctly. + // The second entry is AT_NULL, which marks the end of the vector + let auxv = vec![ + Elf64_Auxv { + a_type: AT_ENTRY, + a_val: ctx.entry, + }, + Elf64_Auxv { + a_type: AT_NULL, + a_val: 0, + }, + ]; + + Self { + regions, + threads: vec![thread], + aux_vector: auxv, + } + } +} + +impl ProcessInfoSource for GuestView { + fn pid(&self) -> i32 { + CORE_DUMP_PID + } + fn threads(&self) -> &[elfcore::ThreadView] { + &self.threads + } + fn page_size(&self) -> usize { + CORE_DUMP_PAGE_SIZE + } + fn aux_vector(&self) -> Option<&[elfcore::Elf64_Auxv]> { + Some(&self.aux_vector) + } + fn va_regions(&self) -> &[elfcore::VaRegion] { + &self.regions + } + fn mapped_files(&self) -> Option<&[elfcore::MappedFile]> { + // We don't have mapped files + None + } +} + +/// Structure that reads the guest memory +/// This structure serves as a custom memory reader for `elfcore`'s +/// [`CoreDumpBuilder`] +struct GuestMemReader { + regions: Vec, +} + +impl GuestMemReader { + fn new(ctx: &CrashDumpContext) -> Self { + Self { + regions: ctx.regions.to_vec(), + } + } +} + +impl ReadProcessMemory for GuestMemReader { + fn read_process_memory( + &mut self, + base: usize, + buf: &mut [u8], + ) -> std::result::Result { + for r in self.regions.iter() { + // Check if the base address is within the guest region + if base >= r.guest_region.start && base < r.guest_region.end { + let offset = base - r.guest_region.start; + let region_slice = unsafe { + std::slice::from_raw_parts( + r.host_region.start as *const u8, + r.host_region.len(), + ) + }; + + // Calculate how much we can copy + let copy_size = min(buf.len(), region_slice.len() - offset); + if copy_size == 0 { + return std::result::Result::Ok(0); + } + + // Only copy the amount that fits in both buffers + buf[..copy_size].copy_from_slice(®ion_slice[offset..offset + copy_size]); + + // Return the number of bytes copied + return std::result::Result::Ok(copy_size); + } + } + + // If we reach here, we didn't find a matching region + std::result::Result::Ok(0) + } +} + +/// Create core dump file from the hypervisor information if the sandbox is configured +/// to allow core dumps. +/// +/// This function generates an ELF core dump file capturing the hypervisor's state, +/// which can be used for debugging when crashes occur. +/// The location of the core dump file is determined by the `HYPERLIGHT_CORE_DUMP_DIR` +/// environment variable. If not set, it defaults to the system's temporary directory. +/// +/// # Arguments +/// * `hv`: Reference to the hypervisor implementation +/// +/// # Returns +/// * `Result<()>`: Success or error +pub(crate) fn generate_crashdump(hv: &dyn Hypervisor) -> Result<()> { + log::info!("Creating core dump file..."); + + // Get crash context from hypervisor + let ctx = hv + .crashdump_context() + .map_err(|e| new_error!("Failed to get crashdump context: {:?}", e))?; + + // Get env variable for core dump directory + let core_dump_dir = std::env::var("HYPERLIGHT_CORE_DUMP_DIR").ok(); + + // Compute file path on the filesystem + let file_path = core_dump_file_path(core_dump_dir); + + let create_dump_file = || { + // Create the file + Ok(Box::new( + std::fs::File::create(&file_path) + .map_err(|e| new_error!("Failed to create core dump file: {:?}", e))?, + ) as Box) + }; + + checked_core_dump(ctx, create_dump_file).map(|_| { + println!("Core dump created successfully: {}", file_path); + log::error!("Core dump file: {}", file_path); + }) +} + +/// Computes the file path for the core dump file. +/// +/// The file path is generated based on the current timestamp and an +/// output directory. +/// If the directory does not exist, it falls back to the system's temp directory. +/// If the variable is not set, it defaults to the system's temporary directory. +/// The filename is formatted as `hl_core_.elf`. +/// +/// Arguments: +/// * `dump_dir`: The environment variable value to check for the output directory. +/// +/// Returns: +/// * `String`: The file path for the core dump file. +fn core_dump_file_path(dump_dir: Option) -> String { + // Generate timestamp string for the filename using chrono + let timestamp = chrono::Local::now().format("%Y%m%d_%H%M%S").to_string(); + + // Determine the output directory based on environment variable + let output_dir = if let Some(dump_dir) = dump_dir { + // Check if the directory exists + // If it doesn't exist, fall back to the system temp directory + // This is to ensure that the core dump can be created even if the directory is not set + if std::path::Path::new(&dump_dir).exists() { + std::path::PathBuf::from(dump_dir) + } else { + log::warn!( + "Directory \"{}\" does not exist, falling back to temp directory", + dump_dir + ); + std::env::temp_dir() + } + } else { + // Fall back to the system temp directory + std::env::temp_dir() + }; + + // Create the filename with timestamp + let filename = format!("hl_core_{}.elf", timestamp); + let file_path = output_dir.join(filename); + + file_path.to_string_lossy().to_string() +} + +/// Create core dump from Hypervisor context if the sandbox is configured to allow core dumps. +/// +/// Arguments: +/// * `ctx`: Optional crash dump context from the hypervisor. This contains the information +/// needed to create the core dump. If `None`, no core dump will be created. +/// * `get_writer`: Closure that returns a writer to the output destination. +/// +/// Returns: +/// * `Result`: The number of bytes written to the core dump file. +fn checked_core_dump( + ctx: Option, + get_writer: impl FnOnce() -> Result>, +) -> Result { + let mut nbytes = 0; + // If the HV returned a context it means we can create a core dump + // This is the case when the sandbox has been configured at runtime to allow core dumps + if let Some(ctx) = ctx { + // Set up data sources for the core dump + let guest_view = GuestView::new(&ctx); + let memory_reader = GuestMemReader::new(&ctx); + + // Create and write core dump + let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader); + + let writer = get_writer()?; + // Write the core dump directly to the file + nbytes = core_builder + .write(writer) + .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?; + } + + Ok(nbytes) +} + +/// Test module for the crash dump functionality +#[cfg(test)] +mod test { + use super::*; + + /// Test the core_dump_file_path function when the environment variable is set to an existing + /// directory + #[test] + fn test_crashdump_file_path_valid() { + // Get CWD + let valid_dir = std::env::current_dir() + .unwrap() + .to_string_lossy() + .to_string(); + + // Call the function + let path = core_dump_file_path(Some(valid_dir.clone())); + + // Check if the path is correct + assert!(path.contains(&valid_dir)); + } + + /// Test the core_dump_file_path function when the environment variable is set to an invalid + /// directory + #[test] + fn test_crashdump_file_path_invalid() { + // Call the function + let path = core_dump_file_path(Some("/tmp/not_existing_dir".to_string())); + + // Get the temp directory + let temp_dir = std::env::temp_dir().to_string_lossy().to_string(); + + // Check if the path is correct + assert!(path.contains(&temp_dir)); + } + + /// Test the core_dump_file_path function when the environment is not set + /// Check against the default temp directory by using the env::temp_dir() function + #[test] + fn test_crashdump_file_path_default() { + // Call the function + let path = core_dump_file_path(None); + + let temp_dir = std::env::temp_dir().to_string_lossy().to_string(); + + // Check if the path is correct + assert!(path.starts_with(&temp_dir)); } - temp_file.flush()?; - // persist the tempfile to disk - let persist_path = temp_file.path().with_extension("dmp"); - temp_file - .persist(&persist_path) - .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?; + /// Test core is not created when the context is None + #[test] + fn test_crashdump_not_created_when_context_is_none() { + // Call the function with None context + let result = checked_core_dump(None, || Ok(Box::new(std::io::empty()))); - println!("Memory dumped to file: {:?}", persist_path); - log::error!("Memory dumped to file: {:?}", persist_path); + // Check if the result is ok and the number of bytes is 0 + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 0); + } + + /// Test the core dump creation with no regions fails + #[test] + fn test_crashdump_write_fails_when_no_regions() { + // Create a dummy context + let ctx = CrashDumpContext::new( + &[], + [0; 27], + vec![], + 0, + Some("dummy_binary".to_string()), + Some("dummy_filename".to_string()), + ); + + let get_writer = || Ok(Box::new(std::io::empty()) as Box); + + // Call the function + let result = checked_core_dump(Some(ctx), get_writer); + + // Check if the result is an error + // This should fail because there are no regions + assert!(result.is_err()); + } + + /// Check core dump with a dummy region to local vec + /// This test checks if the core dump is created successfully + #[test] + fn test_crashdump_dummy_core_dump() { + let dummy_vec = vec![0; 0x1000]; + let regions = vec![MemoryRegion { + guest_region: 0x1000..0x2000, + host_region: dummy_vec.as_ptr() as usize..dummy_vec.as_ptr() as usize + dummy_vec.len(), + flags: MemoryRegionFlags::READ | MemoryRegionFlags::WRITE, + region_type: crate::mem::memory_region::MemoryRegionType::Code, + }]; + // Create a dummy context + let ctx = CrashDumpContext::new( + ®ions, + [0; 27], + vec![], + 0x1000, + Some("dummy_binary".to_string()), + Some("dummy_filename".to_string()), + ); + + let get_writer = || Ok(Box::new(std::io::empty()) as Box); - Ok(()) + // Call the function + let result = checked_core_dump(Some(ctx), get_writer); + + // Check if the result is ok and the number of bytes is 0 + assert!(result.is_ok()); + // Check the number of bytes written is more than 0x1000 (the size of the region) + assert_eq!(result.unwrap(), 0x2000); + } } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 85dc514b..e442bc48 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -48,6 +48,8 @@ use mshv_bindings::{ }; use mshv_ioctls::{Mshv, VcpuFd, VmFd}; use tracing::{instrument, Span}; +#[cfg(crashdump)] +use {super::crashdump, crate::sandbox::uninitialized::SandboxRuntimeConfig, std::path::Path}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -298,6 +300,8 @@ pub(super) struct HypervLinuxDriver { debug: Option, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] + rt_cfg: SandboxRuntimeConfig, } impl HypervLinuxDriver { @@ -316,6 +320,7 @@ impl HypervLinuxDriver { rsp_ptr: GuestPtr, pml4_ptr: GuestPtr, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); @@ -395,6 +400,8 @@ impl HypervLinuxDriver { debug, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg, }) } @@ -679,8 +686,61 @@ impl Hypervisor for HypervLinuxDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set up the registers for the crash dump + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // Get the filename from the binary path + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave.buffer.to_vec(), + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } #[cfg(gdb)] @@ -800,6 +860,15 @@ mod tests { pml4_ptr, #[cfg(gdb)] None, + #[cfg(crashdump)] + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path: None, + #[cfg(gdb)] + debug_info: None, + #[cfg(crashdump)] + guest_core_dump: true, + }, ) .unwrap(); } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index abc2b77c..6e3042bc 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -27,6 +27,8 @@ use windows::Win32::System::Hypervisor::{ WHV_MEMORY_ACCESS_TYPE, WHV_PARTITION_HANDLE, WHV_REGISTER_VALUE, WHV_RUN_VP_EXIT_CONTEXT, WHV_RUN_VP_EXIT_REASON, WHV_X64_SEGMENT_REGISTER, WHV_X64_SEGMENT_REGISTER_0, }; +#[cfg(crashdump)] +use {super::crashdump, crate::sandbox::uninitialized::SandboxRuntimeConfig, std::path::Path}; use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -56,6 +58,8 @@ pub(crate) struct HypervWindowsDriver { entrypoint: u64, orig_rsp: GuestPtr, mem_regions: Vec, + #[cfg(crashdump)] + rt_cfg: SandboxRuntimeConfig, } /* This does not automatically impl Send/Sync because the host * address of the shared memory region is a raw pointer, which are @@ -66,6 +70,7 @@ unsafe impl Send for HypervWindowsDriver {} unsafe impl Sync for HypervWindowsDriver {} impl HypervWindowsDriver { + #[allow(clippy::too_many_arguments)] #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] pub(crate) fn new( mem_regions: Vec, @@ -75,6 +80,7 @@ impl HypervWindowsDriver { entrypoint: u64, rsp: u64, mmap_file_handle: HandleWrapper, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { // create and setup hypervisor partition let mut partition = VMPartition::new(1)?; @@ -102,6 +108,8 @@ impl HypervWindowsDriver { entrypoint, orig_rsp: GuestPtr::try_from(RawPtr::from(rsp))?, mem_regions, + #[cfg(crashdump)] + rt_cfg, }) } @@ -491,8 +499,61 @@ impl Hypervisor for HypervWindowsDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.processor.get_regs()?; + let sregs = self.processor.get_sregs()?; + let xsave = self.processor.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = unsafe { sregs.cs.Segment.Selector } as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = unsafe { sregs.ss.Segment.Selector } as u64; // ss + regs[21] = unsafe { sregs.fs.Segment.Base }; // fs_base + regs[22] = unsafe { sregs.gs.Segment.Base }; // gs_base + regs[23] = unsafe { sregs.ds.Segment.Selector } as u64; // ds + regs[24] = unsafe { sregs.es.Segment.Selector } as u64; // es + regs[25] = unsafe { sregs.fs.Segment.Selector } as u64; // fs + regs[26] = unsafe { sregs.gs.Segment.Selector } as u64; // gs + + // Get the filename from the config + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave, + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } } diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index 8e351708..91d23982 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -51,6 +51,8 @@ use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory, SharedMemory}; #[cfg(gdb)] use crate::sandbox::config::DebugInfo; use crate::sandbox::hypervisor::{get_available_hypervisor, HypervisorType}; +#[cfg(any(crashdump, gdb))] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; #[cfg(target_os = "linux")] use crate::signal_handlers::setup_signal_handlers; use crate::HyperlightError::{ @@ -237,7 +239,7 @@ impl HypervisorHandler { pub(crate) fn start_hypervisor_handler( &mut self, sandbox_memory_manager: SandboxMemoryManager, - #[cfg(gdb)] debug_info: Option, + #[cfg(any(crashdump, gdb))] rt_cfg: SandboxRuntimeConfig, ) -> Result<()> { let configuration = self.configuration.clone(); @@ -299,8 +301,8 @@ impl HypervisorHandler { { hv = Some(set_up_hypervisor_partition( execution_variables.shm.try_lock().map_err(|e| new_error!("Failed to lock shm: {}", e))?.deref_mut().as_mut().ok_or_else(|| new_error!("shm not set"))?, - #[cfg(gdb)] - &debug_info, + #[cfg(any(crashdump, gdb))] + &rt_cfg, )?); } let hv = hv.as_mut().ok_or_else(|| new_error!("Hypervisor not set"))?; @@ -821,7 +823,7 @@ pub enum HandlerMsg { fn set_up_hypervisor_partition( mgr: &mut SandboxMemoryManager, - #[cfg(gdb)] debug_info: &Option, + #[cfg(any(crashdump, gdb))] rt_cfg: &SandboxRuntimeConfig, ) -> Result> { let mem_size = u64::try_from(mgr.shared_mem.mem_size())?; let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?; @@ -857,8 +859,8 @@ fn set_up_hypervisor_partition( // Create gdb thread if gdb is enabled and the configuration is provided #[cfg(gdb)] - let gdb_conn = if let Some(DebugInfo { port }) = debug_info { - let gdb_conn = create_gdb_thread(*port, unsafe { pthread_self() }); + let gdb_conn = if let Some(DebugInfo { port }) = rt_cfg.debug_info { + let gdb_conn = create_gdb_thread(port, unsafe { pthread_self() }); // in case the gdb thread creation fails, we still want to continue // without gdb @@ -884,6 +886,8 @@ fn set_up_hypervisor_partition( pml4_ptr, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg.clone(), )?; Ok(Box::new(hv)) } @@ -897,6 +901,8 @@ fn set_up_hypervisor_partition( rsp_ptr.absolute()?, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg.clone(), )?; Ok(Box::new(hv)) } @@ -914,6 +920,8 @@ fn set_up_hypervisor_partition( entrypoint_ptr.absolute()?, rsp_ptr.absolute()?, HandleWrapper::from(mmap_file_handle), + #[cfg(crashdump)] + rt_cfg.clone(), )?; Ok(Box::new(hv)) } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 3dd1cb1f..eee79374 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -24,6 +24,8 @@ use kvm_ioctls::Cap::UserMemory; use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use log::LevelFilter; use tracing::{instrument, Span}; +#[cfg(crashdump)] +use {super::crashdump, crate::sandbox::uninitialized::SandboxRuntimeConfig, std::path::Path}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -286,6 +288,8 @@ pub(super) struct KVMDriver { debug: Option, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] + rt_cfg: SandboxRuntimeConfig, } impl KVMDriver { @@ -299,6 +303,7 @@ impl KVMDriver { entrypoint: u64, rsp: u64, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { let kvm = Kvm::new()?; @@ -350,6 +355,8 @@ impl KVMDriver { debug, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg, }; Ok(ret) @@ -587,8 +594,67 @@ impl Hypervisor for KVMDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // Get the filename from the runtime config + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + // The [`CrashDumpContext`] accepts xsave as a vector of u8, so we need to convert the + // xsave region to a vector of u8 + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave + .region + .iter() + .flat_map(|item| item.to_le_bytes()) + .collect::>(), + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 62cebe82..5dc3158d 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -232,7 +232,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion]; + fn crashdump_context(&self) -> Result>; #[cfg(gdb)] /// handles the cases when the vCPU stops due to a Debug event @@ -275,7 +275,7 @@ impl VirtualCPU { } Ok(HyperlightExit::Mmio(addr)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; mem_access_fn .clone() @@ -287,7 +287,7 @@ impl VirtualCPU { } Ok(HyperlightExit::AccessViolation(addr, tried, region_permission)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; if region_permission.intersects(MemoryRegionFlags::STACK_GUARD) { return Err(HyperlightError::StackOverflow()); @@ -313,14 +313,14 @@ impl VirtualCPU { } Ok(HyperlightExit::Unknown(reason)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => continue, Err(e) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; return Err(e); } @@ -347,6 +347,8 @@ pub(crate) mod tests { }; use crate::mem::ptr::RawPtr; use crate::sandbox::uninitialized::GuestBinary; + #[cfg(any(crashdump, gdb))] + use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::sandbox::{SandboxConfiguration, UninitializedSandbox}; use crate::{new_error, Result}; @@ -405,8 +407,15 @@ pub(crate) mod tests { hv_handler.start_hypervisor_handler( gshm, - #[cfg(gdb)] - None, + #[cfg(any(crashdump, gdb))] + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path: None, + #[cfg(gdb)] + debug_info: None, + #[cfg(crashdump)] + guest_core_dump: true, + }, )?; hv_handler.execute_hypervisor_handler_action(HypervisorHandlerAction::Initialise) diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index 14fd86b1..a46b02d9 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -26,6 +26,8 @@ use windows_result::HRESULT; use super::wrappers::HandleWrapper; use crate::hypervisor::wrappers::{WHvFPURegisters, WHvGeneralRegisters, WHvSpecialRegisters}; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; +#[cfg(crashdump)] +use crate::HyperlightError; use crate::{new_error, Result}; /// Interop calls for Windows Hypervisor Platform APIs @@ -409,6 +411,59 @@ impl VMProcessor { } } + #[cfg(crashdump)] + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + pub(super) fn get_xsave(&self) -> Result> { + // Get the required buffer size by calling with NULL buffer + let mut buffer_size_needed: u32 = 0; + + unsafe { + // First call with NULL buffer to get required size + // If the buffer is not large enough, the return value is WHV_E_INSUFFICIENT_BUFFER. + // In this case, BytesWritten receives the required buffer size. + let result = WHvGetVirtualProcessorXsaveState( + self.get_partition_hdl(), + 0, + std::ptr::null_mut(), + 0, + &mut buffer_size_needed, + ); + + // If it failed for reasons other than insufficient buffer, return error + if let Err(e) = result { + if e.code() != windows::Win32::Foundation::WHV_E_INSUFFICIENT_BUFFER { + return Err(HyperlightError::WindowsAPIError(e)); + } + } + } + + // Create a buffer with the appropriate size + let mut xsave_buffer = vec![0; buffer_size_needed as usize]; + + // Get the Xsave state + let mut written_bytes = 0; + unsafe { + WHvGetVirtualProcessorXsaveState( + self.get_partition_hdl(), + 0, + xsave_buffer.as_mut_ptr() as *mut std::ffi::c_void, + buffer_size_needed, + &mut written_bytes, + ) + }?; + + // Check if the number of written bytes matches the expected size + if written_bytes != buffer_size_needed { + return Err(new_error!( + "Failed to get Xsave state: expected {} bytes, got {}", + buffer_size_needed, + written_bytes + )); + } + + Ok(xsave_buffer) + } + pub(super) fn set_fpu(&mut self, regs: &WHvFPURegisters) -> Result<()> { const LEN: usize = 26; diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index 5c9f30f2..39e5849b 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -33,6 +33,14 @@ pub struct DebugInfo { #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] pub struct SandboxConfiguration { + /// Guest core dump output directory + /// This field is by default set to true which means the value core dumps will be placed in: + /// - HYPERLIGHT_CORE_DUMP_DIR environment variable if it is set + /// - default value of the temporary directory + /// + /// The core dump files generation can be disabled by setting this field to false. + #[cfg(crashdump)] + guest_core_dump: bool, /// Guest gdb debug port #[cfg(gdb)] guest_debug_info: Option, @@ -123,6 +131,7 @@ impl SandboxConfiguration { max_initialization_time: Option, max_wait_for_cancellation: Option, #[cfg(gdb)] guest_debug_info: Option, + #[cfg(crashdump)] guest_core_dump: bool, ) -> Self { Self { input_data_size: max(input_data_size, Self::MIN_INPUT_SIZE), @@ -178,6 +187,8 @@ impl SandboxConfiguration { }, #[cfg(gdb)] guest_debug_info, + #[cfg(crashdump)] + guest_core_dump, } } @@ -263,6 +274,15 @@ impl SandboxConfiguration { } } + /// Toggles the guest core dump generation for a sandbox + /// Setting this to false disables the core dump generation + /// This is only used when the `crashdump` feature is enabled + #[cfg(crashdump)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub fn set_guest_core_dump(&mut self, enable: bool) { + self.guest_core_dump = enable; + } + /// Sets the configuration for the guest debug #[cfg(gdb)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] @@ -294,6 +314,12 @@ impl SandboxConfiguration { self.max_initialization_time } + #[cfg(crashdump)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub(crate) fn get_guest_core_dump(&self) -> bool { + self.guest_core_dump + } + #[cfg(gdb)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub(crate) fn get_guest_debug_info(&self) -> Option { @@ -340,6 +366,8 @@ impl Default for SandboxConfiguration { None, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ) } } @@ -374,6 +402,8 @@ mod tests { )), #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ); let exe_info = simple_guest_exe_info().unwrap(); @@ -417,6 +447,8 @@ mod tests { )), #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ); assert_eq!(SandboxConfiguration::MIN_INPUT_SIZE, cfg.input_data_size); assert_eq!(SandboxConfiguration::MIN_OUTPUT_SIZE, cfg.output_data_size); diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index 30c136c1..ced79809 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -23,8 +23,6 @@ use std::time::Duration; use log::LevelFilter; use tracing::{instrument, Span}; -#[cfg(gdb)] -use super::config::DebugInfo; use super::host_funcs::{default_writer_func, FunctionRegistry}; use super::mem_mgr::MemMgrWrapper; use super::uninitialized_evolve::evolve_impl_multi_use; @@ -55,6 +53,17 @@ const EXTRA_ALLOWED_SYSCALLS_FOR_WRITER_FUNC: &[super::ExtraAllowedSyscall] = &[ libc::SYS_close, ]; +#[cfg(any(crashdump, gdb))] +#[derive(Clone, Debug, Default)] +pub(crate) struct SandboxRuntimeConfig { + #[cfg(crashdump)] + pub(crate) binary_path: Option, + #[cfg(gdb)] + pub(crate) debug_info: Option, + #[cfg(crashdump)] + pub(crate) guest_core_dump: bool, +} + /// A preliminary `Sandbox`, not yet ready to execute guest code. /// /// Prior to initializing a full-fledged `Sandbox`, you must create one of @@ -71,8 +80,8 @@ pub struct UninitializedSandbox { pub(crate) max_execution_time: Duration, pub(crate) max_wait_for_cancellation: Duration, pub(crate) max_guest_log_level: Option, - #[cfg(gdb)] - pub(crate) debug_info: Option, + #[cfg(any(crashdump, gdb))] + pub(crate) rt_cfg: SandboxRuntimeConfig, } impl crate::sandbox_state::sandbox::UninitializedSandbox for UninitializedSandbox { @@ -152,20 +161,43 @@ impl UninitializedSandbox { GuestBinary::FilePath(binary_path) => { let path = Path::new(&binary_path) .canonicalize() - .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))?; - GuestBinary::FilePath( - path.into_os_string() - .into_string() - .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?, - ) + .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))? + .into_os_string() + .into_string() + .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?; + + GuestBinary::FilePath(path) } buffer @ GuestBinary::Buffer(_) => buffer, }; let sandbox_cfg = cfg.unwrap_or_default(); - #[cfg(gdb)] - let debug_info = sandbox_cfg.get_guest_debug_info(); + #[cfg(any(crashdump, gdb))] + let rt_cfg = { + #[cfg(crashdump)] + let guest_core_dump = sandbox_cfg.get_guest_core_dump(); + + #[cfg(gdb)] + let debug_info = sandbox_cfg.get_guest_debug_info(); + + #[cfg(crashdump)] + let binary_path = if let GuestBinary::FilePath(ref path) = guest_binary { + Some(path.clone()) + } else { + None + }; + + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path, + #[cfg(gdb)] + debug_info, + #[cfg(crashdump)] + guest_core_dump, + } + }; + let mut mem_mgr_wrapper = { let mut mgr = UninitializedSandbox::load_guest_binary(sandbox_cfg, &guest_binary)?; let stack_guard = Self::create_stack_guard(); @@ -188,8 +220,8 @@ impl UninitializedSandbox { sandbox_cfg.get_max_wait_for_cancellation() as u64, ), max_guest_log_level: None, - #[cfg(gdb)] - debug_info, + #[cfg(any(crashdump, gdb))] + rt_cfg, }; // If we were passed a writer for host print register it otherwise use the default. diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs index 05c44579..9af1dc01 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs @@ -23,14 +23,14 @@ use tracing::{instrument, Span}; #[cfg(gdb)] use super::mem_access::dbg_mem_access_handler_wrapper; +#[cfg(any(crashdump, gdb))] +use super::uninitialized::SandboxRuntimeConfig; use crate::hypervisor::hypervisor_handler::{ HvHandlerConfig, HypervisorHandler, HypervisorHandlerAction, }; use crate::mem::mgr::SandboxMemoryManager; use crate::mem::ptr::RawPtr; use crate::mem::shared_mem::GuestSharedMemory; -#[cfg(gdb)] -use crate::sandbox::config::DebugInfo; use crate::sandbox::host_funcs::FunctionRegistry; use crate::sandbox::mem_access::mem_access_handler_wrapper; use crate::sandbox::outb::outb_handler_wrapper; @@ -72,8 +72,8 @@ where u_sbox.max_execution_time, u_sbox.max_wait_for_cancellation, u_sbox.max_guest_log_level, - #[cfg(gdb)] - u_sbox.debug_info, + #[cfg(any(crashdump, gdb))] + u_sbox.rt_cfg, )?; { @@ -110,7 +110,7 @@ fn hv_init( max_exec_time: Duration, max_wait_for_cancellation: Duration, max_guest_log_level: Option, - #[cfg(gdb)] debug_info: Option, + #[cfg(any(crashdump, gdb))] rt_cfg: SandboxRuntimeConfig, ) -> Result { let outb_hdl = outb_handler_wrapper(hshm.clone(), host_funcs); let mem_access_hdl = mem_access_handler_wrapper(hshm.clone()); @@ -147,8 +147,8 @@ fn hv_init( hv_handler.start_hypervisor_handler( gshm, - #[cfg(gdb)] - debug_info, + #[cfg(any(crashdump, gdb))] + rt_cfg, )?; hv_handler