Enable traditional TLB-shootdowns on aarch64

kernel/captain/Cargo.toml

@@ -12,6 +12,7 @@ irq_safety = { git = "https://github.com/theseus-os/irq_safety" }
 dfqueue = { path = "../../libs/dfqueue", version = "0.1.0" }
 multicore_bringup = { path = "../multicore_bringup" }
 early_printer = { path = "../early_printer" }
+tlb_shootdown = { path = "../tlb_shootdown" }
 kernel_config = { path = "../kernel_config" }
 interrupts = { path = "../interrupts" }
 scheduler = { path = "../scheduler" }
@@ -29,7 +30,6 @@ logger_x86_64 = { path = "../logger_x86_64" }
 window_manager = { path = "../window_manager" }
 first_application = { path = "../first_application" }
 exceptions_full = { path = "../exceptions_full" }
-tlb_shootdown = { path = "../tlb_shootdown" }
 multiple_heaps = { path = "../multiple_heaps" }
 tsc = { path = "../tsc" }
 acpi = { path = "../acpi" }

kernel/captain/src/lib.rs

@@ -150,8 +150,6 @@ pub fn init(
 
     // Now that other CPUs are fully booted, init TLB shootdowns,
     // which rely on Local APICs to broadcast an IPI to all running CPUs.
-    // arch-gate: no multicore support on aarch64 at the moment
-    #[cfg(target_arch = "x86_64")]
     tlb_shootdown::init();
 
     // Initialize the per-core heaps.

kernel/cpu/src/aarch64.rs

@@ -52,15 +52,16 @@ impl MpidrValue {
     /// Reads an affinity `level` from this `MpidrValue`.
     ///
     /// Panics if the given affinity level is not 0, 1, 2, or 3.
-    pub fn affinity(self, level: u8) -> u8 {
+    pub fn affinity(self, level: u8) -> u64 {
         let shift = match level {
             0 => 0,
             1 => 8,
             2 => 16,
             3 => 32,
             _ => panic!("Valid affinity levels are 0, 1, 2, 3"),
         };
-        (self.0 >> shift) as u8
+
+        self.0 >> shift
     }
 
     /// Create an `MpidrValue` from its four affinity numbers

kernel/gic/Cargo.toml

@@ -12,3 +12,4 @@ zerocopy = "0.5.0"
 log = "0.4.8"
 
 memory = { path = "../memory" }
+cpu = { path = "../cpu" }

kernel/gic/src/gic/cpu_interface_gicv2.rs

@@ -7,17 +7,16 @@
 //! - Sending End-Of-Interrupts signals
 
 use super::GicRegisters;
-use super::U32BYTES;
 use super::Priority;
 use super::InterruptNumber;
 
 mod offset {
-    use super::U32BYTES;
-    pub const CTLR: usize = 0x00 / U32BYTES;
-    pub const PMR:  usize = 0x04 / U32BYTES;
-    pub const IAR:  usize = 0x0C / U32BYTES;
-    pub const RPR:  usize = 0x14 / U32BYTES;
-    pub const EOIR: usize = 0x10 / U32BYTES;
+    use crate::Offset32;
+    pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00);
+    pub(crate) const PMR:  Offset32 = Offset32::from_byte_offset(0x04);
+    pub(crate) const IAR:  Offset32 = Offset32::from_byte_offset(0x0C);
+    pub(crate) const RPR:  Offset32 = Offset32::from_byte_offset(0x14);
+    pub(crate) const EOIR: Offset32 = Offset32::from_byte_offset(0x10);
 }
 
 // enable group 0

kernel/gic/src/gic/cpu_interface_gicv3.rs

@@ -8,17 +8,17 @@
 //! - Generating software interrupts
 
 use core::arch::asm;
-use super::TargetCpu;
+use super::IpiTargetCpu;
 use super::Priority;
 use super::InterruptNumber;
 
-const SGIR_TARGET_ALL_OTHER_PE: usize = 1 << 40;
-const IGRPEN_ENABLED: usize = 1;
+const SGIR_TARGET_ALL_OTHER_PE: u64 = 1 << 40;
+const IGRPEN_ENABLED: u64 = 1;
 
 /// Enables routing of group 1 interrupts for the current CPU and configures
 /// the end-of-interrupt mode
 pub fn init() {
-    let mut icc_ctlr: usize;
+    let mut icc_ctlr: u64;
 
     unsafe { asm!("mrs {}, ICC_CTLR_EL1", out(reg) icc_ctlr) };
     // clear bit 1 (EOIMode) so that eoi signals both
@@ -40,7 +40,7 @@ pub fn init() {
 /// until this CPU or another one is ready to handle
 /// them
 pub fn get_minimum_priority() -> Priority {
-    let mut reg_value: usize;
+    let mut reg_value: u64;
     unsafe { asm!("mrs {}, ICC_PMR_EL1", out(reg) reg_value) };
     u8::MAX - (reg_value as u8)
 }
@@ -50,15 +50,15 @@ pub fn get_minimum_priority() -> Priority {
 /// until this CPU or another one is ready to handle
 /// them
 pub fn set_minimum_priority(priority: Priority) {
-    let reg_value = (u8::MAX - priority) as usize;
+    let reg_value = (u8::MAX - priority) as u64;
     unsafe { asm!("msr ICC_PMR_EL1, {}", in(reg) reg_value) };
 }
 
 /// Signals to the controller that the currently processed interrupt has
 /// been fully handled, by zeroing the current priority level of this CPU.
 /// This implies that the CPU is ready to process interrupts again.
 pub fn end_of_interrupt(int: InterruptNumber) {
-    let reg_value = int as usize;
+    let reg_value = int as u64;
     unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) };
 }
 
@@ -67,8 +67,8 @@ pub fn end_of_interrupt(int: InterruptNumber) {
 /// the requested interrupt is being handled by
 /// this CPU.
 pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) {
-    let int_num: usize;
-    let priority: usize;
+    let int_num: u64;
+    let priority: u64;
 
     // Reading the interrupt number has the side effect
     // of acknowledging the interrupt.
@@ -82,26 +82,26 @@ pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) {
     (int_num as InterruptNumber, priority as u8)
 }
 
-pub fn send_ipi(int_num: InterruptNumber, target: TargetCpu) {
+pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) {
     let mut value = match target {
-        TargetCpu::Specific(cpu) => {
-            let cpu = cpu as usize;
+        IpiTargetCpu::Specific(cpu) => {
+            let mpidr: cpu::MpidrValue = cpu.into();
 
             // level 3 affinity is expected in cpu[24:31]
             // we want it in bits [48:55]
-            let aff3 = (cpu & 0xff000000) << 24;
+            let aff3 = mpidr.affinity(3) << 48;
 
             // level 2 affinity is expected in cpu[16:23]
             // we want it in bits [32:39]
-            let aff2 = cpu & 0xff0000 << 16;
+            let aff2 = mpidr.affinity(2) << 32;
 
             // level 1 affinity is expected in cpu[8:15]
             // we want it in bits [16:23]
-            let aff1 = cpu & 0xff00 << 8;
+            let aff1 = mpidr.affinity(1) << 16;
 
             // level 0 affinity is expected in cpu[0:7]
             // we want it as a GICv2-style target list
-            let aff0 = cpu & 0xff;
+            let aff0 = mpidr.affinity(0);
             let target_list = if aff0 >= 16 {
                 panic!("[GIC driver] cannot send an IPI to a core with Aff0 >= 16");
             } else {
@@ -111,12 +111,12 @@ pub fn send_ipi(int_num: InterruptNumber, target: TargetCpu) {
         },
         // bit 31: Interrupt Routing Mode
         // value of 1 to target any available cpu
-        TargetCpu::AnyCpuAvailable => SGIR_TARGET_ALL_OTHER_PE,
-        TargetCpu::GICv2TargetList(_) => {
-            panic!("Cannot use TargetCpu::GICv2TargetList with GICv3!");
+        IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE,
+        IpiTargetCpu::GICv2TargetList(_) => {
+            panic!("Cannot use IpiTargetCpu::GICv2TargetList with GICv3!");
         },
     };
 
-    value |= (int_num as usize) << 24;
+    value |= (int_num as u64) << 24;
     unsafe { asm!("msr ICC_SGI1R_EL1, {}", in(reg) value) };
 }

kernel/gic/src/gic/dist_interface.rs

@@ -13,22 +13,22 @@
 //! - Generating software interrupts (GICv2 style)
 
 use super::GicRegisters;
-use super::U32BYTES;
-use super::TargetCpu;
+use super::SpiDestination;
+use super::IpiTargetCpu;
 use super::InterruptNumber;
 use super::Enabled;
 use super::TargetList;
 
 mod offset {
-    use super::U32BYTES;
-    pub const CTLR:      usize = 0x000 / U32BYTES;
-    pub const IGROUPR:   usize = 0x080 / U32BYTES;
-    pub const ISENABLER: usize = 0x100 / U32BYTES;
-    pub const ICENABLER: usize = 0x180 / U32BYTES;
-    pub const ITARGETSR: usize = 0x800 / U32BYTES;
-    pub const SGIR:      usize = 0xf00 / U32BYTES;
+    use crate::{Offset32, Offset64};
+    pub(crate) const CTLR:      Offset32 = Offset32::from_byte_offset(0x000);
+    pub(crate) const IGROUPR:   Offset32 = Offset32::from_byte_offset(0x080);
+    pub(crate) const ISENABLER: Offset32 = Offset32::from_byte_offset(0x100);
+    pub(crate) const ICENABLER: Offset32 = Offset32::from_byte_offset(0x180);
+    pub(crate) const ITARGETSR: Offset32 = Offset32::from_byte_offset(0x800);
+    pub(crate) const SGIR:      Offset32 = Offset32::from_byte_offset(0xf00);
     /// This one is on the 6th page
-    pub const P6IROUTER: usize = 0x100 / U32BYTES;
+    pub(crate) const P6IROUTER: Offset64 = Offset64::from_byte_offset(0x100);
 }
 
 // enable group 0
@@ -48,20 +48,14 @@ const SGIR_TARGET_ALL_OTHER_PE: u32 = 1 << 24;
 // bit 31: SPI routing
 //   1 = any available PE
 //   0 = route to specific PE
-const P6IROUTER_ANY_AVAILABLE_PE: u32 = 1 << 31;
+const P6IROUTER_ANY_AVAILABLE_PE: u64 = 1 << 31;
 
 // const GROUP_0: u32 = 0;
 const GROUP_1: u32 = 1;
 
 // bit 15: which interrupt group to target
 const SGIR_NSATT_GRP1: u32 = 1 << 15;
 
-fn assert_cpu_bounds(target: &TargetCpu) {
-    if let TargetCpu::Specific(cpu) = target {
-        assert!(*cpu < 8, "affinity routing is disabled; cannot target a CPU with id >= 8");
-    }
-}
-
 /// Initializes the distributor by enabling forwarding
 /// of group 1 interrupts
 ///
@@ -106,13 +100,15 @@ pub fn enable_spi(registers: &mut GicRegisters, int: InterruptNumber, enabled: E
 ///
 /// legacy / GICv2 method
 /// int_num must be less than 16
-pub fn send_ipi_gicv2(registers: &mut GicRegisters, int_num: u32, target: TargetCpu) {
-    assert_cpu_bounds(&target);
+pub fn send_ipi_gicv2(registers: &mut GicRegisters, int_num: u32, target: IpiTargetCpu) {
+    if let IpiTargetCpu::Specific(cpu) = target {
+        assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8");
+    }
 
     let target_list = match target {
-        TargetCpu::Specific(cpu) => (1 << cpu) << 16,
-        TargetCpu::AnyCpuAvailable => SGIR_TARGET_ALL_OTHER_PE,
-        TargetCpu::GICv2TargetList(list) => (list.bits as u32) << 16,
+        IpiTargetCpu::Specific(cpu) => (1 << cpu.value()) << 16,
+        IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE,
+        IpiTargetCpu::GICv2TargetList(list) => (list.bits as u32) << 16,
     };
 
     let value: u32 = int_num | target_list | SGIR_NSATT_GRP1;
@@ -139,35 +135,37 @@ impl super::ArmGic {
     /// to a specific CPU or to any PE that is ready
     /// to process them, i.e. not handling another
     /// higher-priority interrupt.
-    pub fn get_spi_target(&self, int: InterruptNumber) -> TargetCpu {
+    pub fn get_spi_target(&self, int: InterruptNumber) -> SpiDestination {
         assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU");
         if !self.affinity_routing() {
             let flags = self.distributor().read_array_volatile::<4>(offset::ITARGETSR, int);
             if flags == 0xff {
-                return TargetCpu::AnyCpuAvailable;
+                return SpiDestination::AnyCpuAvailable;
             }
 
             for i in 0..8 {
                 let target = 1 << i;
                 if target & flags == target {
-                    return TargetCpu::Specific(i);
+                    return SpiDestination::Specific(cpu::MpidrValue::new(0, 0, 0, i).into());
                 }
             }
 
             let list = TargetList::from_bits_truncate(flags as u8);
-            TargetCpu::GICv2TargetList(list)
+            SpiDestination::GICv2TargetList(list)
         } else if let Self::V3(v3) = self {
-            let reg = v3.dist_extended.read_volatile(offset::P6IROUTER);
+            let reg = v3.dist_extended.read_volatile_64(offset::P6IROUTER);
 
             // bit 31: Interrupt Routing Mode
             // value of 1 to target any available cpu
             // value of 0 to target a specific cpu
             if reg & P6IROUTER_ANY_AVAILABLE_PE > 0 {
-                TargetCpu::AnyCpuAvailable
+                SpiDestination::AnyCpuAvailable
             } else {
-                let aff3 = (reg >> 8) & 0xff000000;
-                let aff012 = reg & 0xffffff;
-                TargetCpu::Specific(aff3 | aff012)
+                let aff3 = (reg >> 32) as u8;
+                let aff2 = (reg >> 16) as u8;
+                let aff1 = (reg >>  8) as u8;
+                let aff0 = (reg >>  0) as u8;
+                SpiDestination::Specific(cpu::MpidrValue::new(aff3, aff2, aff1, aff0).into())
             }
         } else {
             // If we're on gicv2 then affinity routing is off
@@ -181,38 +179,32 @@ impl super::ArmGic {
     /// to a specific CPU or to any PE that is ready
     /// to process them, i.e. not handling another
     /// higher-priority interrupt.
-    pub fn set_spi_target(&mut self, int: InterruptNumber, target: TargetCpu) {
+    pub fn set_spi_target(&mut self, int: InterruptNumber, target: SpiDestination) {
         assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU");
         if !self.affinity_routing() {
-            assert_cpu_bounds(&target);
+            if let SpiDestination::Specific(cpu) = target {
+                assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8");
+            }
 
             let value = match target {
-                TargetCpu::Specific(cpu) => 1 << cpu,
-                TargetCpu::AnyCpuAvailable => 0xff,
-                TargetCpu::GICv2TargetList(list) => list.bits as u32,
+                SpiDestination::Specific(cpu) => 1 << cpu.value(),
+                SpiDestination::AnyCpuAvailable => 0xff,
+                SpiDestination::GICv2TargetList(list) => list.bits as u32,
             };
 
             self.distributor_mut().write_array_volatile::<4>(offset::ITARGETSR, int, value);
         } else if let Self::V3(v3) = self {
             let value = match target {
-                TargetCpu::Specific(cpu) => {
-                    // shift aff3 8 bits to the left
-                    let aff3 = (cpu & 0xff000000) << 8;
-                    // keep aff 0, 1 & 2 where they are
-                    let aff012 = cpu & 0xffffff;
-                    // leave bit 31 clear to indicate
-                    // a specific target
-                    aff3 | aff012
-                },
+                SpiDestination::Specific(cpu) => cpu::MpidrValue::from(cpu).value(),
                 // bit 31: Interrupt Routing Mode
                 // value of 1 to target any available cpu
-                TargetCpu::AnyCpuAvailable => P6IROUTER_ANY_AVAILABLE_PE,
-                TargetCpu::GICv2TargetList(_) => {
-                    panic!("Cannot use TargetCpu::GICv2TargetList with GICv3!");
+                SpiDestination::AnyCpuAvailable => P6IROUTER_ANY_AVAILABLE_PE,
+                SpiDestination::GICv2TargetList(_) => {
+                    panic!("Cannot use SpiDestination::GICv2TargetList with GICv3!");
                 },
             };
 
-            v3.dist_extended.write_volatile(offset::P6IROUTER, value);
+            v3.dist_extended.write_volatile_64(offset::P6IROUTER, value);
         }
 
         // If we're on gicv2 then affinity routing is off