monolithic: enter user space

Cargo.toml

@@ -70,6 +70,8 @@ members = [
     "apps/task/yield",
     "apps/task/priority",
     "apps/task/tls",
+
+    "variants/monolithic",
 ]
 
 [profile.release]

modules/axhal/Cargo.toml

@@ -16,12 +16,14 @@ fp_simd = []
 paging = ["axalloc", "page_table"]
 irq = []
 tls = ["alloc"]
+uspace = ["paging"]
 default = []
 
 [dependencies]
 log = "0.4"
 cfg-if = "1.0"
 bitflags = "2.2"
+spin = "0.9"
 static_assertions = "1.1.0"
 axlog = { path = "../axlog" }
 axconfig = { path = "../axconfig" }

modules/axhal/src/arch/riscv/trap.S

@@ -42,7 +42,7 @@
 trap_vector_base:
     // sscratch == 0: trap from S mode
     // sscratch != 0: trap from U mode
-    csrrw   sp, sscratch, sp            // switch sscratch and sp
+    csrrw   sp, sscratch, sp            // swap sscratch and sp
     bnez    sp, .Ltrap_entry_u
 
     csrr    sp, sscratch                // put supervisor sp back

modules/axhal/src/arch/x86_64/context.rs

@@ -1,10 +1,16 @@
 use core::{arch::asm, fmt};
 use memory_addr::VirtAddr;
 
+#[cfg(feature = "irq")]
+use x86_64::registers::rflags::RFlags;
+
+#[cfg(feature = "uspace")]
+use super::gdt::GdtStruct;
+
 /// Saved registers when a trap (interrupt or exception) occurs.
 #[allow(missing_docs)]
 #[repr(C)]
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Copy)]
 pub struct TrapFrame {
     pub rax: u64,
     pub rcx: u64,
@@ -41,6 +47,60 @@ impl TrapFrame {
     }
 }
 
+#[cfg(feature = "uspace")]
+pub struct UspaceContext(TrapFrame);
+
+#[cfg(feature = "uspace")]
+impl UspaceContext {
+    pub fn new(entry: usize, ustack_top: VirtAddr, arg0: usize) -> Self {
+        Self(TrapFrame {
+            rdi: arg0 as _,
+            rip: entry as _,
+            cs: GdtStruct::UCODE64_SELECTOR.0 as _,
+            #[cfg(feature = "irq")]
+            rflags: RFlags::INTERRUPT_FLAG.bits(), // IOPL = 0, IF = 1
+            rsp: ustack_top.as_usize() as _,
+            ss: GdtStruct::UDATA_SELECTOR.0 as _,
+            ..Default::default()
+        })
+    }
+
+    pub const fn from(tf: &TrapFrame) -> Self {
+        let mut tf = *tf;
+        tf.cs = GdtStruct::UCODE64_SELECTOR.0 as _;
+        tf.ss = GdtStruct::UDATA_SELECTOR.0 as _;
+        Self(tf)
+    }
+
+    pub unsafe fn enter_uspace(&self, kstack_top: VirtAddr) -> ! {
+        super::disable_irqs();
+        super::tss_set_rsp0(kstack_top);
+        asm!("
+            mov     rsp, {tf}
+            pop     rax
+            pop     rcx
+            pop     rdx
+            pop     rbx
+            pop     rbp
+            pop     rsi
+            pop     rdi
+            pop     r8
+            pop     r9
+            pop     r10
+            pop     r11
+            pop     r12
+            pop     r13
+            pop     r14
+            pop     r15
+            add     rsp, 16     // skip vector, error_code
+            swapgs
+            iretq",
+            tf = in(reg) &self.0,
+            options(noreturn),
+        )
+    }
+}
+
 #[repr(C)]
 #[derive(Debug, Default)]
 struct ContextSwitchFrame {

modules/axhal/src/arch/x86_64/gdt.rs

@@ -1,11 +1,19 @@
 use core::fmt;
+use spin::Once;
 
 use x86_64::instructions::tables::{lgdt, load_tss};
 use x86_64::registers::segmentation::{Segment, SegmentSelector, CS};
 use x86_64::structures::gdt::{Descriptor, DescriptorFlags};
 use x86_64::structures::{tss::TaskStateSegment, DescriptorTablePointer};
 use x86_64::{addr::VirtAddr, PrivilegeLevel};
 
+#[no_mangle]
+#[percpu::def_percpu]
+static TSS: Once<TaskStateSegment> = Once::new();
+
+#[percpu::def_percpu]
+static GDT: Once<GdtStruct> = Once::new();
+
 /// A wrapper of the Global Descriptor Table (GDT) with maximum 16 entries.
 #[repr(align(16))]
 pub struct GdtStruct {
@@ -86,3 +94,26 @@ impl fmt::Debug for GdtStruct {
             .finish()
     }
 }
+
+/// Initializes the per-CPU TSS and GDT structures and loads them into the
+/// current CPU.
+pub fn init_gdt() {
+    unsafe {
+        let tss = TSS.current_ref_raw();
+        let gdt = GDT.current_ref_raw();
+        let tss = tss.call_once(|| TaskStateSegment::new());
+        let gdt = gdt.call_once(|| GdtStruct::new(tss));
+        gdt.load();
+        gdt.load_tss();
+    }
+}
+
+/// Sets the stack pointer for privilege level 0 (RSP0) of the current TSS.
+///
+/// # Safety
+///
+/// Must be called after initialization and preemption is disabled.
+pub unsafe fn tss_set_rsp0(rsp0: memory_addr::VirtAddr) {
+    let tss = unsafe { TSS.current_ref_mut_raw().get_mut_unchecked() };
+    tss.privilege_stack_table[0] = x86_64::VirtAddr::new(rsp0.as_usize() as u64);
+}

modules/axhal/src/arch/x86_64/idt.rs

@@ -1,11 +1,14 @@
 use core::fmt;
+use spin::Once;
 
 use x86_64::addr::VirtAddr;
 use x86_64::structures::idt::{Entry, HandlerFunc, InterruptDescriptorTable};
 use x86_64::structures::DescriptorTablePointer;
 
 const NUM_INT: usize = 256;
 
+static IDT: Once<IdtStruct> = Once::new();
+
 /// A wrapper of the Interrupt Descriptor Table (IDT).
 #[repr(transparent)]
 pub struct IdtStruct {
@@ -33,7 +36,11 @@ impl IdtStruct {
         };
         for i in 0..NUM_INT {
             #[allow(clippy::missing_transmute_annotations)]
-            entries[i].set_handler_fn(unsafe { core::mem::transmute(ENTRIES[i]) });
+            let opt = entries[i].set_handler_fn(unsafe { core::mem::transmute(ENTRIES[i]) });
+            if i == 0x80 {
+                // enable legacy int 0x80 syscall
+                opt.set_privilege_level(x86_64::PrivilegeLevel::Ring3);
+            }
         }
         idt
     }
@@ -66,3 +73,9 @@ impl fmt::Debug for IdtStruct {
             .finish()
     }
 }
+
+/// Initializes the global IDT and loads it into the current CPU.
+pub fn init_idt() {
+    let idt = IDT.call_once(|| IdtStruct::new());
+    unsafe { idt.load() };
+}

modules/axhal/src/arch/x86_64/mod.rs

@@ -2,6 +2,9 @@ mod context;
 mod gdt;
 mod idt;
 
+#[cfg(feature = "uspace")]
+mod syscall;
+
 #[cfg(target_os = "none")]
 mod trap;
 
@@ -12,9 +15,11 @@ use x86::{controlregs, msr, tlb};
 use x86_64::instructions::interrupts;
 
 pub use self::context::{ExtendedState, FxsaveArea, TaskContext, TrapFrame};
-pub use self::gdt::GdtStruct;
-pub use self::idt::IdtStruct;
-pub use x86_64::structures::tss::TaskStateSegment;
+pub use self::gdt::{init_gdt, tss_set_rsp0, GdtStruct};
+pub use self::idt::{init_idt, IdtStruct};
+
+#[cfg(feature = "uspace")]
+pub use self::{context::UspaceContext, syscall::init_syscall};
 
 /// Allows the current CPU to respond to interrupts.
 #[inline]

modules/axhal/src/arch/x86_64/syscall.S

@@ -0,0 +1,54 @@
+.section .text
+syscall_entry:
+    swapgs                                                      // switch to kernel gs
+    mov     gs:[offset __PERCPU_USER_RSP_OFFSET], rsp           // save user rsp
+    mov     rsp, gs:[offset __PERCPU_TSS + {tss_rsp0_offset}]   // switch to kernel stack
+
+    sub     rsp, 8                                  // skip user ss
+    push    gs:[offset __PERCPU_USER_RSP_OFFSET]    // user rsp
+    push    r11                                     // rflags
+    mov     [rsp - 2 * 8], rcx                      // rip
+    sub     rsp, 4 * 8                              // skip until general registers
+
+    push    r15
+    push    r14
+    push    r13
+    push    r12
+    push    r11
+    push    r10
+    push    r9
+    push    r8
+    push    rdi
+    push    rsi
+    push    rbp
+    push    rbx
+    push    rdx
+    push    rcx
+    push    rax
+
+    mov     rdi, rsp
+    call    x86_syscall_handler
+
+    pop     rax
+    pop     rcx
+    pop     rdx
+    pop     rbx
+    pop     rbp
+    pop     rsi
+    pop     rdi
+    pop     r8
+    pop     r9
+    pop     r10
+    pop     r11
+    pop     r12
+    pop     r13
+    pop     r14
+    pop     r15
+
+    add     rsp, 7 * 8
+    mov     rcx, [rsp - 5 * 8]  // rip
+    mov     r11, [rsp - 3 * 8]  // rflags
+    mov     rsp, [rsp - 2 * 8]  // user rsp
+
+    swapgs
+    sysretq

modules/axhal/src/arch/x86_64/syscall.rs

@@ -0,0 +1,51 @@
+use x86_64::addr::VirtAddr;
+use x86_64::registers::model_specific::{Efer, EferFlags, KernelGsBase, LStar, SFMask, Star};
+use x86_64::registers::rflags::RFlags;
+use x86_64::structures::tss::TaskStateSegment;
+
+use super::{GdtStruct, TrapFrame};
+
+#[no_mangle]
+#[percpu::def_percpu]
+static USER_RSP_OFFSET: usize = 0;
+
+core::arch::global_asm!(
+    include_str!("syscall.S"),
+    tss_rsp0_offset = const core::mem::offset_of!(TaskStateSegment, privilege_stack_table),
+);
+
+#[no_mangle]
+pub(super) fn x86_syscall_handler(tf: &mut TrapFrame) {
+    info!(
+        "syscall {} [{}, {}, {}, {}]",
+        tf.rax, tf.rdi, tf.rsi, tf.rdx, tf.rdx
+    );
+}
+
+/// Initializes syscall support and setups the syscall handler.
+pub fn init_syscall() {
+    extern "C" {
+        fn syscall_entry();
+    }
+    unsafe {
+        LStar::write(VirtAddr::new(syscall_entry as usize as _));
+        Star::write(
+            GdtStruct::UCODE64_SELECTOR,
+            GdtStruct::UDATA_SELECTOR,
+            GdtStruct::KCODE64_SELECTOR,
+            GdtStruct::KDATA_SELECTOR,
+        )
+        .unwrap();
+        SFMask::write(
+            RFlags::TRAP_FLAG
+                | RFlags::INTERRUPT_FLAG
+                | RFlags::DIRECTION_FLAG
+                | RFlags::IOPL_LOW
+                | RFlags::IOPL_HIGH
+                | RFlags::NESTED_TASK
+                | RFlags::ALIGNMENT_CHECK,
+        ); // TF | IF | DF | IOPL | AC | NT (0x47700)
+        Efer::update(|efer| *efer |= EferFlags::SYSTEM_CALL_EXTENSIONS);
+        KernelGsBase::write(VirtAddr::new(0));
+    }
+}

modules/axhal/src/arch/x86_64/trap.rs

@@ -4,11 +4,14 @@ use super::context::TrapFrame;
 
 core::arch::global_asm!(include_str!("trap.S"));
 
+#[cfg(feature = "uspace")]
+const LEGACY_SYSCALL_VECTOR: u8 = 0x80;
+
 const IRQ_VECTOR_START: u8 = 0x20;
 const IRQ_VECTOR_END: u8 = 0xff;
 
 #[no_mangle]
-fn x86_trap_handler(tf: &TrapFrame) {
+fn x86_trap_handler(tf: &mut TrapFrame) {
     match tf.vector as u8 {
         PAGE_FAULT_VECTOR => {
             if tf.is_user() {
@@ -35,12 +38,26 @@ fn x86_trap_handler(tf: &TrapFrame) {
                 tf.rip, tf.error_code, tf
             );
         }
+        #[cfg(feature = "uspace")]
+        LEGACY_SYSCALL_VECTOR => super::syscall::x86_syscall_handler(tf),
         IRQ_VECTOR_START..=IRQ_VECTOR_END => crate::trap::handle_irq_extern(tf.vector as _),
         _ => {
             panic!(
-                "Unhandled exception {} (error_code = {:#x}) @ {:#x}:\n{:#x?}",
-                tf.vector, tf.error_code, tf.rip, tf
+                "Unhandled exception {} ({}, error_code = {:#x}) @ {:#x}:\n{:#x?}",
+                tf.vector,
+                vec_to_str(tf.vector),
+                tf.error_code,
+                tf.rip,
+                tf
             );
         }
     }
 }
+
+fn vec_to_str(vec: u64) -> &'static str {
+    if vec < 32 {
+        EXCEPTIONS[vec as usize].mnemonic
+    } else {
+        "Unknown"
+    }
+}