diff --git a/src/core/src/vmm/aarch64.rs b/src/core/src/vmm/aarch64.rs index c79d5c328..47b8aa5b3 100644 --- a/src/core/src/vmm/aarch64.rs +++ b/src/core/src/vmm/aarch64.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::hv::{Cpu, CpuFeats, CpuStates, Pstate}; +use super::hv::{Cpu, CpuFeats, CpuStates, Pstate, Sctlr, Tcr}; use super::ram::RamMap; use super::MainCpuError; +use std::sync::atomic::Ordering; pub fn setup_main_cpu( cpu: &mut impl Cpu, @@ -9,6 +10,9 @@ pub fn setup_main_cpu( map: RamMap, feats: &CpuFeats, ) -> Result<(), MainCpuError> { + // Acquire the memory modified by RAM builder. + std::sync::atomic::fence(Ordering::Acquire); + // Check if CPU support VM page size. let mut states = cpu .states() @@ -41,23 +45,37 @@ pub fn setup_main_cpu( ); // Enable MMU to enable virtual address and set TCR_EL1. - states.set_sctlr_el1(true); + states.set_sctlr( + Sctlr::new() + .with_m(true) + .with_c(true) + .with_itd(true) + .with_i(true) + .with_tscxt(true) + .with_span(true) + .with_ntlsmd(true) + .with_lsmaoe(true), + ); states.set_mair_el1(map.memory_attrs); - states.set_tcr_el1( - true, // Ignore tob-byte when translate address with TTBR1_EL1. - true, // Ignore top-byte when translate address with TTBR0_EL1. - 0b101, // 48 bits Intermediate Physical Address. - match map.page_size.get() { - 0x4000 => 0b01, // 16K page for TTBR1_EL1. - _ => todo!(), - }, - false, // Use ASID from TTBR0_EL1. - 16, // 48-bit virtual addresses for TTBR1_EL1. - match map.page_size.get() { - 0x4000 => 0b10, // 16K page for TTBR0_EL1. - _ => todo!(), - }, - 16, // 48-bit virtual addresses for TTBR0_EL1. + states.set_tcr( + Tcr::new() + .with_ips(feats.mmfr0.pa_range()) + .with_tg1(match map.page_size.get() { + 0x4000 => 0b01, // 16K page for TTBR1_EL1. + _ => todo!(), + }) + .with_sh1(0b11) + .with_orgn1(0b01) + .with_irgn1(0b01) + .with_t1sz(16) + .with_tg0(match map.page_size.get() { + 0x4000 => 0b10, // 16K page for TTBR0_EL1. + _ => todo!(), + }) + .with_sh0(0b11) + .with_orgn0(0b01) + .with_irgn0(0b01) + .with_t0sz(16), ); // Set page table. We need both lower and higher VA here because the virtual devices mapped with diff --git a/src/core/src/vmm/hv/aarch64.rs b/src/core/src/vmm/hv/aarch64.rs index 731180ce5..8ed18e1b1 100644 --- a/src/core/src/vmm/hv/aarch64.rs +++ b/src/core/src/vmm/hv/aarch64.rs @@ -26,6 +26,133 @@ pub struct Pstate { __: u32, } +/// Represents a value of `SCTLR_EL1`. +#[bitfield(u64)] +pub struct Sctlr { + pub m: bool, + pub a: bool, + pub c: bool, + pub sa: bool, + pub sa0: bool, + pub cp15ben: bool, + pub naa: bool, + pub itd: bool, + pub sed: bool, + pub uma: bool, + pub enrctx: bool, + pub eos: bool, + pub i: bool, + pub endb: bool, + pub dze: bool, + pub uct: bool, + pub ntwi: bool, + __: bool, + pub ntwe: bool, + pub wxn: bool, + pub tscxt: bool, + pub iesb: bool, + pub eis: bool, + pub span: bool, + pub e0e: bool, + pub ee: bool, + pub uci: bool, + pub enda: bool, + pub ntlsmd: bool, + pub lsmaoe: bool, + pub enib: bool, + pub enia: bool, + pub cmow: bool, + pub mscen: bool, + __: bool, + pub bt0: bool, + pub bt1: bool, + pub itfsb: bool, + #[bits(2)] + pub tcf0: u8, + #[bits(2)] + pub tcf: u8, + pub ata0: bool, + pub ata: bool, + pub dssbs: bool, + pub tweden: bool, + #[bits(4)] + pub twedel: u8, + pub tmt0: bool, + pub tmt: bool, + pub tme0: bool, + pub tme: bool, + pub enasr: bool, + pub enas0: bool, + pub enals: bool, + pub epan: bool, + pub tcso0: bool, + pub tcso: bool, + pub entp2: bool, + pub nmi: bool, + pub spintmask: bool, + pub tidcp: bool, +} + +/// Represents a value of `TCR_EL1`. +#[bitfield(u64)] +pub struct Tcr { + #[bits(6)] + pub t0sz: u8, + __: bool, + pub epd0: bool, + #[bits(2)] + pub irgn0: u8, + #[bits(2)] + pub orgn0: u8, + #[bits(2)] + pub sh0: u8, + #[bits(2)] + pub tg0: u8, + #[bits(6)] + pub t1sz: u8, + pub a1: bool, + pub epd1: bool, + #[bits(2)] + pub irgn1: u8, + #[bits(2)] + pub orgn1: u8, + #[bits(2)] + pub sh1: u8, + #[bits(2)] + pub tg1: u8, + #[bits(3)] + pub ips: u8, + __: bool, + pub asid: bool, + pub tbi0: bool, + pub tbi1: bool, + pub ha: bool, + pub hd: bool, + pub hpd0: bool, + pub hpd1: bool, + pub hwu059: bool, + pub hwu060: bool, + pub hwu061: bool, + pub hwu062: bool, + pub hwu159: bool, + pub hwu160: bool, + pub hwu161: bool, + pub hwu162: bool, + pub tbid0: bool, + pub tbid1: bool, + pub nfd0: bool, + pub nfd1: bool, + pub e0pd0: bool, + pub e0pd1: bool, + pub tcma0: bool, + pub tcma1: bool, + pub ds: bool, + pub mtx0: bool, + pub mtx1: bool, + __: bool, + __: bool, +} + /// Represents a value of `ID_AA64MMFR0_EL1`. /// /// All documentation copied from Arm Architecture Reference Manual for A-profile architecture. diff --git a/src/core/src/vmm/hv/macos/cpu.rs b/src/core/src/vmm/hv/macos/cpu.rs index f145aca82..5db0580ee 100644 --- a/src/core/src/vmm/hv/macos/cpu.rs +++ b/src/core/src/vmm/hv/macos/cpu.rs @@ -131,9 +131,9 @@ impl<'a> Cpu for HfCpu<'a> { Ok(HfStates { cpu: self, pstate: State::None, - sctlr_el1: State::None, + sctlr: State::None, mair_el1: State::None, - tcr_el1: State::None, + tcr: State::None, ttbr0_el1: State::None, ttbr1_el1: State::None, sp_el1: State::None, @@ -214,11 +214,11 @@ pub struct HfStates<'a, 'b> { #[cfg(target_arch = "aarch64")] pstate: State, #[cfg(target_arch = "aarch64")] - sctlr_el1: State, + sctlr: State, #[cfg(target_arch = "aarch64")] mair_el1: State, #[cfg(target_arch = "aarch64")] - tcr_el1: State, + tcr: State, #[cfg(target_arch = "aarch64")] ttbr0_el1: State, #[cfg(target_arch = "aarch64")] @@ -312,11 +312,8 @@ impl<'a, 'b> CpuStates for HfStates<'a, 'b> { } #[cfg(target_arch = "aarch64")] - fn set_sctlr_el1(&mut self, m: bool) { - // All hard-coded values came from https://github.com/AsahiLinux/m1n1/issues/97/ - let m: u64 = m.into(); - - self.sctlr_el1 = State::Dirty(0x30901084 | m); + fn set_sctlr(&mut self, v: crate::vmm::hv::Sctlr) { + self.sctlr = State::Dirty(v.into_bits()); } #[cfg(target_arch = "aarch64")] @@ -325,42 +322,8 @@ impl<'a, 'b> CpuStates for HfStates<'a, 'b> { } #[cfg(target_arch = "aarch64")] - fn set_tcr_el1( - &mut self, - tbi1: bool, - tbi0: bool, - ips: u8, - tg1: u8, - a1: bool, - t1sz: u8, - tg0: u8, - t0sz: u8, - ) { - let tbi1: u64 = tbi1.into(); - let tbi0: u64 = tbi0.into(); - let ips: u64 = ips.into(); - let tg1: u64 = tg1.into(); - let a1: u64 = a1.into(); - let t1sz: u64 = t1sz.into(); - let tg0: u64 = tg0.into(); - let t0sz: u64 = t0sz.into(); - - assert_eq!(ips & 0b11111000, 0); - assert_eq!(tg1 & 0b11111100, 0); - assert_eq!(t1sz & 0b11000000, 0); - assert_eq!(tg0 & 0b11111100, 0); - assert_eq!(t0sz & 0b11000000, 0); - - self.tcr_el1 = State::Dirty( - tbi1 << 38 - | tbi0 << 37 - | ips << 32 - | tg1 << 30 - | a1 << 22 - | t1sz << 16 - | tg0 << 14 - | t0sz, - ); + fn set_tcr(&mut self, v: crate::vmm::hv::Tcr) { + self.tcr = State::Dirty(v.into_bits()); } #[cfg(target_arch = "aarch64")] @@ -470,12 +433,12 @@ impl<'a, 'b> CpuStates for HfStates<'a, 'b> { set_sys(HV_SYS_REG_TTBR1_EL1, v).map_err(StatesError::SetTtbr1El1Failed)?; } - if let State::Dirty(v) = self.tcr_el1 { - set_sys(HV_SYS_REG_TCR_EL1, v).map_err(StatesError::SetTcrEl1Failed)?; + if let State::Dirty(v) = self.tcr { + set_sys(HV_SYS_REG_TCR_EL1, v).map_err(StatesError::SetTcrFailed)?; } - if let State::Dirty(v) = self.sctlr_el1 { - set_sys(HV_SYS_REG_SCTLR_EL1, v).map_err(StatesError::SetSctlrEl1Failed)?; + if let State::Dirty(v) = self.sctlr { + set_sys(HV_SYS_REG_SCTLR_EL1, v).map_err(StatesError::SetSctlrFailed)?; } if let State::Dirty(v) = self.sp_el1 { @@ -545,11 +508,11 @@ pub enum StatesError { #[cfg(target_arch = "aarch64")] #[error("couldn't set SCTLR_EL1")] - SetSctlrEl1Failed(NonZero), + SetSctlrFailed(NonZero), #[cfg(target_arch = "aarch64")] #[error("couldn't set TCR_EL1")] - SetTcrEl1Failed(NonZero), + SetTcrFailed(NonZero), #[cfg(target_arch = "aarch64")] #[error("couldn't set MAIR_EL1")] diff --git a/src/core/src/vmm/hv/mod.rs b/src/core/src/vmm/hv/mod.rs index 935a52627..60bfad316 100644 --- a/src/core/src/vmm/hv/mod.rs +++ b/src/core/src/vmm/hv/mod.rs @@ -95,27 +95,13 @@ pub trait CpuStates { fn set_pstate(&mut self, v: Pstate); #[cfg(target_arch = "aarch64")] - fn set_sctlr_el1(&mut self, m: bool); + fn set_sctlr(&mut self, v: Sctlr); #[cfg(target_arch = "aarch64")] fn set_mair_el1(&mut self, attrs: u64); - /// # Panics - /// - If `ips` greater than 7. - /// - If `tg1` or `tg0` geater than 3. - /// - If `t1sz` or `t0sz` larger than 6 bits. #[cfg(target_arch = "aarch64")] - fn set_tcr_el1( - &mut self, - tbi1: bool, - tbi0: bool, - ips: u8, - tg1: u8, - a1: bool, - t1sz: u8, - tg0: u8, - t0sz: u8, - ); + fn set_tcr(&mut self, v: Tcr); /// # Panics /// If `baddr` has non-zero on bit 0 or 48:64. diff --git a/src/core/src/vmm/mod.rs b/src/core/src/vmm/mod.rs index 9a5959f63..d16a004a8 100644 --- a/src/core/src/vmm/mod.rs +++ b/src/core/src/vmm/mod.rs @@ -406,7 +406,7 @@ pub unsafe extern "C" fn vmm_run( } // Build RAM. - let map = match ram.build(vm_page_size, &devices, dynamic) { + let map = match ram.build(&feats, vm_page_size, &devices, dynamic) { Ok(v) => v, Err(e) => { *err = RustError::with_source("couldn't build RAM", e); diff --git a/src/core/src/vmm/ram/builder.rs b/src/core/src/vmm/ram/builder.rs index da892bc45..2aced1fcc 100644 --- a/src/core/src/vmm/ram/builder.rs +++ b/src/core/src/vmm/ram/builder.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{Ram, RamError}; -use crate::vmm::hv::Hypervisor; +use crate::vmm::hv::{CpuFeats, Hypervisor}; use crate::vmm::hw::DeviceTree; use crate::vmm::kernel::ProgramHeader; use obconf::{BootEnv, Config}; @@ -175,6 +175,7 @@ impl<'a> RamBuilder<'a> { impl<'a> RamBuilder<'a> { pub fn build( mut self, + _: &CpuFeats, page_size: NonZero, devices: &DeviceTree, dynamic: ProgramHeader, @@ -360,24 +361,29 @@ impl<'a> RamBuilder<'a> { pub fn build( mut self, + feats: &CpuFeats, page_size: NonZero, devices: &DeviceTree, dynamic: ProgramHeader, ) -> Result { // Setup page tables. let map = match page_size.get() { - 0x4000 => self.build_16k_page_tables(devices)?, + 0x4000 => self.build_16k_page_tables(feats, devices)?, _ => todo!(), }; // Relocate the kernel to virtual address. unsafe { self.relocate_kernel(&map, dynamic, 1027)? }; + // Flush modified memory. + std::sync::atomic::fence(std::sync::atomic::Ordering::Release); + Ok(map) } fn build_16k_page_tables( &mut self, + feats: &CpuFeats, devices: &DeviceTree, ) -> Result { // Allocate page table level 0. @@ -395,7 +401,7 @@ impl<'a> RamBuilder<'a> { for (addr, dev) in devices.map() { let len = dev.len().get(); - self.setup_16k_page_tables(l0t, addr, addr, len, Self::MA_DEV_NG_NR_NE)?; + self.setup_16k_page_tables(feats, l0t, addr, addr, len, Self::MA_DEV_NG_NR_NE)?; dev_end = addr + len; } @@ -411,7 +417,7 @@ impl<'a> RamBuilder<'a> { assert!(vaddr >= dev_end); - self.setup_16k_page_tables(l0t, vaddr, kern_paddr, kern_len, Self::MA_NOR)?; + self.setup_16k_page_tables(feats, l0t, vaddr, kern_paddr, kern_len, Self::MA_NOR)?; vaddr += kern_len; @@ -423,7 +429,7 @@ impl<'a> RamBuilder<'a> { .map(|v| (v.start, v.end - v.start)) .unwrap(); - self.setup_16k_page_tables(l0t, vaddr, paddr, stack_len, Self::MA_NOR)?; + self.setup_16k_page_tables(feats, l0t, vaddr, paddr, stack_len, Self::MA_NOR)?; vaddr += stack_len; @@ -431,8 +437,16 @@ impl<'a> RamBuilder<'a> { let args = self.args.take().unwrap(); let ram = args.ram; let env_vaddr = vaddr + args.env; + let conf_vaddr = vaddr + args.conf; - self.setup_16k_page_tables(l0t, vaddr, ram.start, ram.end - ram.start, Self::MA_NOR)?; + self.setup_16k_page_tables( + feats, + l0t, + vaddr, + ram.start, + ram.end - ram.start, + Self::MA_NOR, + )?; Ok(RamMap { page_size: unsafe { NonZero::new_unchecked(0x4000) }, @@ -444,12 +458,13 @@ impl<'a> RamBuilder<'a> { stack_vaddr, stack_len, env_vaddr, - conf_vaddr: todo!(), + conf_vaddr, }) } fn setup_16k_page_tables( &mut self, + _: &CpuFeats, l0t: &mut [usize; 32], vaddr: usize, paddr: usize, @@ -461,12 +476,12 @@ impl<'a> RamBuilder<'a> { assert_eq!(len % 0x4000, 0); assert_eq!(attr & 0b11111000, 0); - fn set_page_entry(entry: &mut usize, addr: usize) { + fn set_table_descriptor(entry: &mut usize, addr: usize) { assert_eq!(addr & 0xFFFF000000003FFF, 0); *entry = addr; - *entry |= 0b01; // Valid. - *entry |= 0b10; // Table descriptor/Page descriptor. + *entry |= 0b11; // Valid + Table descriptor/Page descriptor + *entry |= 1 << 10; // AF } for off in (0..len).step_by(0x4000) { @@ -479,7 +494,7 @@ impl<'a> RamBuilder<'a> { .alloc_16k_page_table() .map_err(RamBuilderError::AllocPageTableLevel1Failed)?; - set_page_entry(&mut l0t[l0o], addr); + set_table_descriptor(&mut l0t[l0o], addr); unsafe { &mut *l1t } } @@ -494,7 +509,7 @@ impl<'a> RamBuilder<'a> { .alloc_16k_page_table() .map_err(RamBuilderError::AllocPageTableLevel2Failed)?; - set_page_entry(&mut l1t[l1o], addr); + set_table_descriptor(&mut l1t[l1o], addr); unsafe { &mut *l2t } } @@ -509,7 +524,7 @@ impl<'a> RamBuilder<'a> { .alloc_16k_page_table() .map_err(RamBuilderError::AllocPageTableLevel3Failed)?; - set_page_entry(&mut l2t[l2o], addr); + set_table_descriptor(&mut l2t[l2o], addr); unsafe { &mut *l3t } } @@ -519,12 +534,18 @@ impl<'a> RamBuilder<'a> { // Set page descriptor. let l3o = (addr & 0x1FFC000) >> 14; let addr = paddr + off; + let mut desc = addr; + assert_eq!(addr & 0xFFFF000000003FFF, 0); assert_eq!(l3t[l3o], 0); - set_page_entry(&mut l3t[l3o], addr); + desc |= 0b11; // Valid descriptor + Page descriptor + desc |= attr << 2; // AttrIndx[2:0] + desc |= 0b00 << 6; // AP[2:1] + desc |= 0b11 << 8; // Inner Shareable + desc |= 1 << 10; // AF - l3t[l3o] |= attr << 2; + l3t[l3o] = desc; } Ok(())