[feat] Add hypervisor support for aarch64

Cargo.toml

@@ -25,6 +25,7 @@ default = []
 fp-simd = []
 tls = []
 uspace = []
+arm-el2 = []
 
 [dependencies]
 linkme = "0.3"

src/aarch64/asm.rs

@@ -46,15 +46,26 @@ pub fn halt() {
 
 /// Reads the current page table root register for kernel space (`TTBR1_EL1`).
 ///
+/// When the "arm-el2" feature is enabled,
+/// TTBR0_EL2 is dedicated to the Hypervisor's Stage-2 page table base address.
+///
 /// Returns the physical address of the page table root.
 #[inline]
 pub fn read_kernel_page_table() -> PhysAddr {
+    #[cfg(not(feature = "arm-el2"))]
     let root = TTBR1_EL1.get();
+
+    #[cfg(feature = "arm-el2")]
+    let root = TTBR0_EL2.get();
+
     pa!(root as usize)
 }
 
 /// Reads the current page table root register for user space (`TTBR0_EL1`).
 ///
+/// When the "arm-el2" feature is enabled, for user-mode programs,
+/// virtualization is completely transparent to them, so there is no need to modify
+///
 /// Returns the physical address of the page table root.
 #[inline]
 pub fn read_user_page_table() -> PhysAddr {
@@ -65,19 +76,33 @@ pub fn read_user_page_table() -> PhysAddr {
 /// Writes the register to update the current page table root for kernel space
 /// (`TTBR1_EL1`).
 ///
+/// When the "arm-el2" feature is enabled,
+/// TTBR0_EL2 is dedicated to the Hypervisor's Stage-2 page table base address.
+///
 /// Note that the TLB is **NOT** flushed after this operation.
 ///
 /// # Safety
 ///
 /// This function is unsafe as it changes the virtual memory address space.
 #[inline]
 pub unsafe fn write_kernel_page_table(root_paddr: PhysAddr) {
-    // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff)
-    TTBR1_EL1.set(root_paddr.as_usize() as _);
+    #[cfg(not(feature = "arm-el2"))]
+    {
+        // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff)
+        TTBR1_EL1.set(root_paddr.as_usize() as _);
+    }
+
+    #[cfg(feature = "arm-el2")]
+    {
+        // kernel space page table at EL2 use TTBR0_EL2 (0x0000_0000_0000_0000..0x0000_ffff_ffff_ffff)
+        TTBR0_EL2.set(root_paddr.as_usize() as _);
+    }
 }
 
 /// Writes the register to update the current page table root for user space
 /// (`TTBR1_EL0`).
+/// When the "arm-el2" feature is enabled, for user-mode programs,
+/// virtualization is completely transparent to them, so there is no need to modify
 ///
 /// Note that the TLB is **NOT** flushed after this operation.
 ///
@@ -97,10 +122,27 @@ pub unsafe fn write_user_page_table(root_paddr: PhysAddr) {
 pub fn flush_tlb(vaddr: Option<VirtAddr>) {
     unsafe {
         if let Some(vaddr) = vaddr {
-            asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) vaddr.as_usize())
+            const VA_MASK: usize = (1 << 44) - 1; // 0xFFFF_FFFF_FFFF
+            let operand = (vaddr.as_usize() >> 12) & VA_MASK;
+
+            #[cfg(not(feature = "arm-el2"))]
+            {
+                asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) operand)
+            }
+            #[cfg(feature = "arm-el2")]
+            {
+                asm!("tlbi vae2is, {}; dsb sy; isb", in(reg) operand)
+            }
         } else {
             // flush the entire TLB
-            asm!("tlbi vmalle1; dsb sy; isb")
+            #[cfg(not(feature = "arm-el2"))]
+            {
+                asm!("tlbi vmalle1; dsb sy; isb")
+            }
+            #[cfg(feature = "arm-el2")]
+            {
+                asm!("tlbi alle2; dsb sy; isb")
+            }
         }
     }
 }
@@ -124,8 +166,11 @@ pub fn flush_dcache_line(vaddr: VirtAddr) {
 /// This function is unsafe as it changes the exception handling behavior of the
 /// current CPU.
 #[inline]
-pub unsafe fn write_exception_vector_base(vbar_el1: usize) {
-    VBAR_EL1.set(vbar_el1 as _);
+pub unsafe fn write_exception_vector_base(vbar: usize) {
+    #[cfg(not(feature = "arm-el2"))]
+    VBAR_EL1.set(vbar as _);
+    #[cfg(feature = "arm-el2")]
+    VBAR_EL2.set(vbar as _);
 }
 
 /// Reads the thread pointer of the current CPU (`TPIDR_EL0`).