diff --git a/src/core/src/vmm/aarch64.rs b/src/core/src/vmm/aarch64.rs new file mode 100644 index 000000000..690f0729f --- /dev/null +++ b/src/core/src/vmm/aarch64.rs @@ -0,0 +1,61 @@ +use super::hv::{Cpu, CpuStates}; +use super::hw::RamMap; +use super::MainCpuError; + +pub fn setup_main_cpu(cpu: &mut impl Cpu, entry: usize, map: RamMap) -> Result<(), MainCpuError> { + // Check if CPU support VM page size. + let mut states = cpu + .states() + .map_err(|e| MainCpuError::GetCpuStatesFailed(Box::new(e)))?; + let mmfr0 = states + .get_id_aa64_mmfr0() + .map_err(|e| MainCpuError::GetIdAa64mmfr0Failed(Box::new(e)))?; + + match map.page_size.get() { + 0x4000 => { + if ((mmfr0 & 0xF00000) >> 20) == 0 { + return Err(MainCpuError::PageSizeNotSupported(map.page_size)); + } + } + _ => todo!(), + } + + // Set PSTATE so the PE run in AArch64 mode. Not sure why we need M here since the document said + // it is ignore. See https://gist.github.com/imbushuo/51b09e61ecd7b7ac063853ad65cedf34 where + // M = 5 came from. + states.set_pstate(true, true, true, true, 0b101); + + // Enable MMU to enable virtual address and set TCR_EL1. + states.set_sctlr_el1(true); + states.set_mair_el1(map.memory_attrs); + states.set_tcr_el1( + true, // Ignore tob-byte when translate address with TTBR1_EL1. + true, // Ignore top-byte when translate address with TTBR0_EL1. + 0b101, // 48 bits Intermediate Physical Address. + match map.page_size.get() { + 0x4000 => 0b01, // 16K page for TTBR1_EL1. + _ => todo!(), + }, + false, // Use ASID from TTBR0_EL1. + 16, // 48-bit virtual addresses for TTBR1_EL1. + match map.page_size.get() { + 0x4000 => 0b10, // 16K page for TTBR0_EL1. + _ => todo!(), + }, + 16, // 48-bit virtual addresses for TTBR0_EL1. + ); + + // Set page table. We need both lower and higher VA here because the virtual devices mapped with + // identity mapping. + states.set_ttbr0_el1(map.page_table); + states.set_ttbr1_el1(map.page_table); + + // Set entry point, its argument and stack pointer. + states.set_x0(map.env_vaddr); + states.set_sp_el1(map.stack_vaddr + map.stack_len); // Top-down. + states.set_pc(map.kern_vaddr + entry); + + states + .commit() + .map_err(|e| MainCpuError::CommitCpuStatesFailed(Box::new(e))) +} diff --git a/src/core/src/vmm/hv/macos/cpu.rs b/src/core/src/vmm/hv/macos/cpu.rs index 2cc325a65..2adb39c65 100644 --- a/src/core/src/vmm/hv/macos/cpu.rs +++ b/src/core/src/vmm/hv/macos/cpu.rs @@ -1,8 +1,8 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 use super::arch::HfExit; use crate::vmm::hv::{Cpu, CpuStates}; use hv_sys::hv_vcpu_destroy; use std::marker::PhantomData; -use std::mem::MaybeUninit; use std::num::NonZero; use thiserror::Error; @@ -58,12 +58,24 @@ impl<'a> HfCpu<'a> { } } + #[cfg(target_arch = "aarch64")] + fn read_sys(&self, reg: hv_sys::hv_sys_reg_t) -> Result> { + use hv_sys::hv_vcpu_get_sys_reg; + + let mut v = 0; + + match NonZero::new(unsafe { hv_vcpu_get_sys_reg(self.instance, reg, &mut v) }) { + Some(v) => Err(v), + None => Ok(v), + } + } + #[cfg(target_arch = "x86_64")] fn read_register( &self, register: hv_sys::hv_x86_reg_t, ) -> Result> { - let mut value = MaybeUninit::::uninit(); + let mut value = std::mem::MaybeUninit::::uninit(); wrap_return!(unsafe { hv_sys::hv_vcpu_read_register(self.instance, register, value.as_mut_ptr().cast()) @@ -117,6 +129,7 @@ impl<'a> Cpu for HfCpu<'a> { fn states(&mut self) -> Result, Self::GetStatesErr> { Ok(HfStates { cpu: self, + id_aa64_mmfr0: State::None, pstate: State::None, sctlr_el1: State::None, mair_el1: State::None, @@ -176,6 +189,8 @@ impl<'a> Drop for HfCpu<'a> { /// Implementation of [`Cpu::States`] for Hypervisor Framework. pub struct HfStates<'a, 'b> { cpu: &'a mut HfCpu<'b>, + #[cfg(target_arch = "aarch64")] + id_aa64_mmfr0: State, #[cfg(target_arch = "x86_64")] rsp: State, #[cfg(target_arch = "x86_64")] @@ -221,6 +236,25 @@ pub struct HfStates<'a, 'b> { impl<'a, 'b> CpuStates for HfStates<'a, 'b> { type Err = StatesError; + #[cfg(target_arch = "aarch64")] + fn get_id_aa64_mmfr0(&mut self) -> Result { + use hv_sys::hv_sys_reg_t_HV_SYS_REG_ID_AA64MMFR0_EL1 as HV_SYS_REG_ID_AA64MMFR0_EL1; + + let v = match self.id_aa64_mmfr0 { + State::None => { + let v = self + .cpu + .read_sys(HV_SYS_REG_ID_AA64MMFR0_EL1) + .map_err(StatesError::ReadRegisterFailed)?; + self.id_aa64_mmfr0 = State::Clean(v); + v + } + State::Clean(v) | State::Dirty(v) => v, + }; + + Ok(v) + } + #[cfg(target_arch = "x86_64")] fn set_rdi(&mut self, v: usize) { todo!() @@ -505,6 +539,10 @@ pub enum RunError { /// Implementation of [`Cpu::GetStatesErr`] and [`CpuStates::Err`]. #[derive(Debug, Error)] pub enum StatesError { + #[cfg(target_arch = "aarch64")] + #[error("couldn't read the register")] + ReadRegisterFailed(NonZero), + #[cfg(target_arch = "x86_64")] #[error("couldn't set RIP")] SetRipFailed(NonZero), diff --git a/src/core/src/vmm/hv/mod.rs b/src/core/src/vmm/hv/mod.rs index fc20a27cb..b3873c4c9 100644 --- a/src/core/src/vmm/hv/mod.rs +++ b/src/core/src/vmm/hv/mod.rs @@ -58,6 +58,9 @@ pub trait Cpu { pub trait CpuStates { type Err: Error + Send + 'static; + #[cfg(target_arch = "aarch64")] + fn get_id_aa64_mmfr0(&mut self) -> Result; + #[cfg(target_arch = "x86_64")] fn set_rdi(&mut self, v: usize); diff --git a/src/core/src/vmm/mod.rs b/src/core/src/vmm/mod.rs index a96da1dfb..c1334c5ae 100644 --- a/src/core/src/vmm/mod.rs +++ b/src/core/src/vmm/mod.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use self::hv::{Cpu, CpuExit, CpuIo, CpuStates, Hypervisor}; +use self::hv::{Cpu, CpuExit, CpuIo, Hypervisor}; use self::hw::{setup_devices, Device, DeviceContext, DeviceTree, Ram, RamBuilder, RamMap}; use self::kernel::{ Kernel, PT_DYNAMIC, PT_GNU_EH_FRAME, PT_GNU_RELRO, PT_GNU_STACK, PT_LOAD, PT_NOTE, PT_PHDR, @@ -21,6 +21,9 @@ use std::sync::Arc; use std::thread::JoinHandle; use thiserror::Error; +#[cfg_attr(target_arch = "aarch64", path = "aarch64.rs")] +#[cfg_attr(target_arch = "x86_64", path = "x86_64.rs")] +mod arch; mod hv; mod hw; mod kernel; @@ -391,7 +394,7 @@ pub unsafe extern "C" fn vmm_run( // Setup hypervisor. let ram = Arc::new(ram); let hv = match self::hv::new(8, ram.clone()) { - Ok(v) => Arc::new(v), + Ok(v) => v, Err(e) => { *err = RustError::with_source("couldn't setup a hypervisor", e); return null_mut(); @@ -480,7 +483,7 @@ fn main_cpu( } }; - if let Err(e) = setup_main_cpu(&mut cpu, entry, map) { + if let Err(e) = self::arch::setup_main_cpu(&mut cpu, entry, map) { status.send(Err(e)).unwrap(); return; } @@ -492,109 +495,6 @@ fn main_cpu( run_cpu(cpu, args); } -#[cfg(target_arch = "x86_64")] -fn setup_main_cpu(cpu: &mut impl Cpu, entry: usize, map: RamMap) -> Result<(), MainCpuError> { - // Set CR3 to page-map level-4 table. - let mut states = cpu - .states() - .map_err(|e| MainCpuError::GetCpuStatesFailed(Box::new(e)))?; - - assert_eq!(map.page_table & 0xFFF0000000000FFF, 0); - - states.set_cr3(map.page_table); - - // Set CR4. - let mut cr4 = 0; - - cr4 |= 0x20; // Physical-address extensions (PAE). - - states.set_cr4(cr4); - - // Set EFER. - let mut efer = 0; - - efer |= 0x100; // Long Mode Enable (LME). - efer |= 0x400; // Long Mode Active (LMA). - - states.set_efer(efer); - - // Set CR0. - let mut cr0 = 0; - - cr0 |= 0x00000001; // Protected Mode Enable (PE). - cr0 |= 0x80000000; // Paging (PG). - - states.set_cr0(cr0); - - // Set CS to 64-bit mode with ring 0. Although x86-64 specs from AMD ignore the Code/Data flag - // on 64-bit mode but Intel CPU violate this spec so we need to enable it. - states.set_cs(0b1000, 0, true, true, false); - - // Set data segments. The only fields used on 64-bit mode is P. - states.set_ds(true); - states.set_es(true); - states.set_fs(true); - states.set_gs(true); - states.set_ss(true); - - // Set entry point, its argument and stack pointer. - states.set_rdi(map.env_vaddr); - states.set_rsp(map.stack_vaddr + map.stack_len); // Top-down. - states.set_rip(map.kern_vaddr + entry); - - if let Err(e) = states.commit() { - return Err(MainCpuError::CommitCpuStatesFailed(Box::new(e))); - } - - Ok(()) -} - -#[cfg(target_arch = "aarch64")] -fn setup_main_cpu(cpu: &mut impl Cpu, entry: usize, map: RamMap) -> Result<(), MainCpuError> { - // Set PSTATE so the PE run in AArch64 mode. Not sure why we need M here since the document said - // it is ignore. See https://gist.github.com/imbushuo/51b09e61ecd7b7ac063853ad65cedf34 where - // M = 5 came from. - let mut states = cpu - .states() - .map_err(|e| MainCpuError::GetCpuStatesFailed(Box::new(e)))?; - - states.set_pstate(true, true, true, true, 0b101); - - // Enable MMU to enable virtual address and set TCR_EL1. - states.set_sctlr_el1(true); - states.set_mair_el1(map.memory_attrs); - states.set_tcr_el1( - true, // Ignore tob-byte when translate address with TTBR1_EL1. - true, // Ignore top-byte when translate address with TTBR0_EL1. - 0b101, // 48 bits Intermediate Physical Address. - match map.page_size.get() { - 0x4000 => 0b01, // 16K page for TTBR1_EL1. - _ => todo!(), - }, - false, // Use ASID from TTBR0_EL1. - 16, // 48-bit virtual addresses for TTBR1_EL1. - match map.page_size.get() { - 0x4000 => 0b10, // 16K page for TTBR0_EL1. - _ => todo!(), - }, - 16, // 48-bit virtual addresses for TTBR0_EL1. - ); - - // Set page table. We need both lower and higher VA here because the virtual devices mapped with - // identity mapping. - states.set_ttbr0_el1(map.page_table); - states.set_ttbr1_el1(map.page_table); - - // Set entry point, its argument and stack pointer. - states.set_x0(map.env_vaddr); - states.set_sp_el1(map.stack_vaddr + map.stack_len); // Top-down. - states.set_pc(map.kern_vaddr + entry); - - states - .commit() - .map_err(|e| MainCpuError::CommitCpuStatesFailed(Box::new(e))) -} - fn run_cpu(mut cpu: C, args: &CpuArgs) { let mut devices = args .devices @@ -753,7 +653,7 @@ impl From for VmmLog { /// Encapsulates arguments for a function to run a CPU. struct CpuArgs { - hv: Arc, + hv: H, ram: Arc, screen: Arc<::Buffer>, devices: Arc, @@ -836,6 +736,14 @@ enum MainCpuError { #[error("couldn't get vCPU states")] GetCpuStatesFailed(#[source] Box), + #[cfg(target_arch = "aarch64")] + #[error("couldn't get ID_AA64MMFR0_EL1")] + GetIdAa64mmfr0Failed(#[source] Box), + + #[cfg(target_arch = "aarch64")] + #[error("vCPU does not support {0:#x} page size")] + PageSizeNotSupported(NonZero), + #[error("couldn't commit vCPU states")] CommitCpuStatesFailed(#[source] Box), } diff --git a/src/core/src/vmm/x86_64.rs b/src/core/src/vmm/x86_64.rs new file mode 100644 index 000000000..e925b2edd --- /dev/null +++ b/src/core/src/vmm/x86_64.rs @@ -0,0 +1,59 @@ +use super::hv::{Cpu, CpuStates}; +use super::hw::RamMap; +use super::MainCpuError; + +pub fn setup_main_cpu(cpu: &mut impl Cpu, entry: usize, map: RamMap) -> Result<(), MainCpuError> { + // Set CR3 to page-map level-4 table. + let mut states = cpu + .states() + .map_err(|e| MainCpuError::GetCpuStatesFailed(Box::new(e)))?; + + assert_eq!(map.page_table & 0xFFF0000000000FFF, 0); + + states.set_cr3(map.page_table); + + // Set CR4. + let mut cr4 = 0; + + cr4 |= 0x20; // Physical-address extensions (PAE). + + states.set_cr4(cr4); + + // Set EFER. + let mut efer = 0; + + efer |= 0x100; // Long Mode Enable (LME). + efer |= 0x400; // Long Mode Active (LMA). + + states.set_efer(efer); + + // Set CR0. + let mut cr0 = 0; + + cr0 |= 0x00000001; // Protected Mode Enable (PE). + cr0 |= 0x80000000; // Paging (PG). + + states.set_cr0(cr0); + + // Set CS to 64-bit mode with ring 0. Although x86-64 specs from AMD ignore the Code/Data flag + // on 64-bit mode but Intel CPU violate this spec so we need to enable it. + states.set_cs(0b1000, 0, true, true, false); + + // Set data segments. The only fields used on 64-bit mode is P. + states.set_ds(true); + states.set_es(true); + states.set_fs(true); + states.set_gs(true); + states.set_ss(true); + + // Set entry point, its argument and stack pointer. + states.set_rdi(map.env_vaddr); + states.set_rsp(map.stack_vaddr + map.stack_len); // Top-down. + states.set_rip(map.kern_vaddr + entry); + + if let Err(e) = states.commit() { + return Err(MainCpuError::CommitCpuStatesFailed(Box::new(e))); + } + + Ok(()) +}