diff --git a/Cargo.lock b/Cargo.lock index 1020a0731..aea972dcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1346,7 +1346,6 @@ name = "gic" version = "0.1.0" dependencies = [ "arm_boards", - "bitflags", "cpu", "log", "memory", diff --git a/kernel/cpu/src/aarch64.rs b/kernel/cpu/src/aarch64.rs index 3fe5128a9..f85abb548 100644 --- a/kernel/cpu/src/aarch64.rs +++ b/kernel/cpu/src/aarch64.rs @@ -6,7 +6,7 @@ use derive_more::{Display, Binary, Octal, LowerHex, UpperHex}; use irq_safety::RwLockIrqSafe; use core::fmt; use alloc::vec::Vec; -use arm_boards::mpidr::DefinedMpidrValue; +use arm_boards::{mpidr::DefinedMpidrValue, BOARD_CONFIG}; use super::CpuId; @@ -71,6 +71,19 @@ pub fn current_cpu() -> CpuId { #[repr(transparent)] pub struct MpidrValue(u64); +/// Affinity Levels and corresponding bit ranges +/// +/// The associated integers are the locations (N..(N+8)) +/// of the corresponding bits in an [`MpidrValue`]. +#[derive(Copy, Clone, Debug)] +#[repr(u64)] +pub enum AffinityShift { + LevelZero = 0, + LevelOne = 8, + LevelTwo = 16, + LevelThree = 32, +} + impl MpidrValue { /// Returns the inner raw value read from the `MPIDR_EL1` register. pub fn value(self) -> u64 { @@ -78,17 +91,8 @@ impl MpidrValue { } /// Reads an affinity `level` from this `MpidrValue`. - /// - /// Panics if the given affinity level is not 0, 1, 2, or 3. - pub fn affinity(self, level: u8) -> u8 { - let shift = match level { - 0 => 0, - 1 => 8, - 2 => 16, - 3 => 32, - _ => panic!("Valid affinity levels are 0, 1, 2, 3"), - }; - (self.0 >> shift) as u8 + pub fn affinity(self, level: AffinityShift) -> u64 { + (self.0 >> (level as u64)) & (u8::MAX as u64) } } @@ -124,6 +128,22 @@ impl From for CpuId { } } +impl TryFrom for MpidrValue { + type Error = &'static str; + + /// Tries to find this MPIDR value in those defined by + /// `arm_boards::cpu_ids`. Fails if No CPU has this MPIDR value. + fn try_from(mpidr_value: u64) -> Result { + for def_mpidr in BOARD_CONFIG.cpu_ids { + if def_mpidr.value() == mpidr_value { + return Ok(def_mpidr.into()) + } + } + + Err("No CPU has this MPIDR value") + } +} + /// An equivalent to Option, which internally encodes None as /// `u32::MAX`, which is an invalid CpuId (bits [4:7] of affinity level /// 0 must always be cleared). This guarantees that it compiles down to diff --git a/kernel/gic/Cargo.toml b/kernel/gic/Cargo.toml index 8a5e5202d..6b37e010c 100644 --- a/kernel/gic/Cargo.toml +++ b/kernel/gic/Cargo.toml @@ -7,7 +7,6 @@ name = "gic" [dependencies] static_assertions = "1.1.0" -bitflags = "1.1.0" zerocopy = "0.5.0" log = "0.4.8" diff --git a/kernel/gic/src/gic/cpu_interface_gicv3.rs b/kernel/gic/src/gic/cpu_interface_gicv3.rs index 3de6a1f7b..b5ee7d399 100644 --- a/kernel/gic/src/gic/cpu_interface_gicv3.rs +++ b/kernel/gic/src/gic/cpu_interface_gicv3.rs @@ -8,17 +8,17 @@ //! - Generating software interrupts use core::arch::asm; -use super::TargetCpu; +use super::IpiTargetCpu; use super::Priority; use super::InterruptNumber; -const SGIR_TARGET_ALL_OTHER_PE: usize = 1 << 40; -const IGRPEN_ENABLED: usize = 1; +const SGIR_TARGET_ALL_OTHER_PE: u64 = 1 << 40; +const IGRPEN_ENABLED: u64 = 1; /// Enables routing of group 1 interrupts for the current CPU and configures /// the end-of-interrupt mode pub fn init() { - let mut icc_ctlr: usize; + let mut icc_ctlr: u64; unsafe { asm!("mrs {}, ICC_CTLR_EL1", out(reg) icc_ctlr) }; // clear bit 1 (EOIMode) so that eoi signals both @@ -40,7 +40,7 @@ pub fn init() { /// until this CPU or another one is ready to handle /// them pub fn get_minimum_priority() -> Priority { - let mut reg_value: usize; + let mut reg_value: u64; unsafe { asm!("mrs {}, ICC_PMR_EL1", out(reg) reg_value) }; u8::MAX - (reg_value as u8) } @@ -50,7 +50,7 @@ pub fn get_minimum_priority() -> Priority { /// until this CPU or another one is ready to handle /// them pub fn set_minimum_priority(priority: Priority) { - let reg_value = (u8::MAX - priority) as usize; + let reg_value = (u8::MAX - priority) as u64; unsafe { asm!("msr ICC_PMR_EL1, {}", in(reg) reg_value) }; } @@ -58,7 +58,7 @@ pub fn set_minimum_priority(priority: Priority) { /// been fully handled, by zeroing the current priority level of this CPU. /// This implies that the CPU is ready to process interrupts again. pub fn end_of_interrupt(int: InterruptNumber) { - let reg_value = int as usize; + let reg_value = int as u64; unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) }; } @@ -67,8 +67,8 @@ pub fn end_of_interrupt(int: InterruptNumber) { /// the requested interrupt is being handled by /// this CPU. pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) { - let int_num: usize; - let priority: usize; + let int_num: u64; + let priority: u64; // Reading the interrupt number has the side effect // of acknowledging the interrupt. @@ -82,41 +82,35 @@ pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) { (int_num as InterruptNumber, priority as u8) } -pub fn send_ipi(int_num: InterruptNumber, target: TargetCpu) { +pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) { let mut value = match target { - TargetCpu::Specific(cpu) => { - let cpu = cpu as usize; - - // level 3 affinity is expected in cpu[24:31] - // we want it in bits [48:55] - let aff3 = (cpu & 0xff000000) << 24; - - // level 2 affinity is expected in cpu[16:23] - // we want it in bits [32:39] - let aff2 = cpu & 0xff0000 << 16; - - // level 1 affinity is expected in cpu[8:15] - // we want it in bits [16:23] - let aff1 = cpu & 0xff00 << 8; - - // level 0 affinity is expected in cpu[0:7] - // we want it as a GICv2-style target list - let aff0 = cpu & 0xff; - let target_list = if aff0 >= 16 { - panic!("[GIC driver] cannot send an IPI to a core with Aff0 >= 16"); - } else { - 1 << aff0 + IpiTargetCpu::Specific(cpu) => { + let mpidr: cpu::MpidrValue = cpu.into(); + + // level 3 affinity in bits [48:55] + let aff3 = mpidr.affinity(cpu::AffinityShift::LevelThree) << 48; + + // level 2 affinity in bits [32:39] + let aff2 = mpidr.affinity(cpu::AffinityShift::LevelTwo) << 32; + + // level 1 affinity in bits [16:23] + let aff1 = mpidr.affinity(cpu::AffinityShift::LevelOne) << 16; + + // level 0 affinity as a GICv2-style target list + let aff0 = mpidr.affinity(cpu::AffinityShift::LevelZero); + let target_list = match aff0 >= 16 { + true => panic!("[GIC driver] cannot send an IPI to a core with Aff0 >= 16"), + false => 1 << aff0, }; + aff3 | aff2 | aff1 | target_list }, - // bit 31: Interrupt Routing Mode - // value of 1 to target any available cpu - TargetCpu::AnyCpuAvailable => SGIR_TARGET_ALL_OTHER_PE, - TargetCpu::GICv2TargetList(_) => { - panic!("Cannot use TargetCpu::GICv2TargetList with GICv3!"); + IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE, + IpiTargetCpu::GICv2TargetList(_) => { + panic!("Cannot use IpiTargetCpu::GICv2TargetList with GICv3!"); }, }; - value |= (int_num as usize) << 24; + value |= (int_num as u64) << 24; unsafe { asm!("msr ICC_SGI1R_EL1, {}", in(reg) value) }; } \ No newline at end of file diff --git a/kernel/gic/src/gic/dist_interface.rs b/kernel/gic/src/gic/dist_interface.rs index 158d72e51..adddb867f 100644 --- a/kernel/gic/src/gic/dist_interface.rs +++ b/kernel/gic/src/gic/dist_interface.rs @@ -13,11 +13,14 @@ //! - Generating software interrupts (GICv2 style) use super::GicRegisters; -use super::TargetCpu; +use super::IpiTargetCpu; +use super::SpiDestination; use super::InterruptNumber; use super::Enabled; use super::TargetList; +use cpu::MpidrValue; + mod offset { use crate::{Offset32, Offset64}; pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x000); @@ -58,12 +61,6 @@ const GROUP_1: u32 = 1; // bit 15: which interrupt group to target const SGIR_NSATT_GRP1: u32 = 1 << 15; -fn assert_cpu_bounds(target: &TargetCpu) { - if let TargetCpu::Specific(cpu) = target { - assert!(*cpu < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); - } -} - /// Initializes the distributor by enabling forwarding /// of group 1 interrupts and allowing the GIC to pick /// a core that is asleep for "1 of N" interrupts. @@ -110,13 +107,15 @@ pub fn enable_spi(registers: &mut GicRegisters, int: InterruptNumber, enabled: E /// /// legacy / GICv2 method /// int_num must be less than 16 -pub fn send_ipi_gicv2(registers: &mut GicRegisters, int_num: u32, target: TargetCpu) { - assert_cpu_bounds(&target); +pub fn send_ipi_gicv2(registers: &mut GicRegisters, int_num: u32, target: IpiTargetCpu) { + if let IpiTargetCpu::Specific(cpu) = &target { + assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); + } let target_list = match target { - TargetCpu::Specific(cpu) => (1 << cpu) << 16, - TargetCpu::AnyCpuAvailable => SGIR_TARGET_ALL_OTHER_PE, - TargetCpu::GICv2TargetList(list) => (list.bits as u32) << 16, + IpiTargetCpu::Specific(cpu) => (1 << cpu.value()) << 16, + IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE, + IpiTargetCpu::GICv2TargetList(list) => (list.0 as u32) << 16, }; let value: u32 = int_num | target_list | SGIR_NSATT_GRP1; @@ -138,28 +137,26 @@ impl super::ArmGic { } } - /// The GIC can be configured to route - /// Shared-Peripheral Interrupts (SPI) either - /// to a specific CPU or to any PE that is ready - /// to process them, i.e. not handling another - /// higher-priority interrupt. - pub fn get_spi_target(&self, int: InterruptNumber) -> TargetCpu { + /// Returns the destination of an SPI if it's valid, i.e. if it + /// points to existing CPU(s). + /// + /// Note: If the destination is a `GICv2TargetList`, the validity + /// of that destination is not checked. + pub fn get_spi_target(&self, int: InterruptNumber) -> Result { assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU"); if !self.affinity_routing() { let flags = self.distributor().read_array_volatile::<4>(offset::ITARGETSR, int); - if flags == 0xff { - return TargetCpu::AnyCpuAvailable; - } for i in 0..8 { let target = 1 << i; - if target & flags == target { - return TargetCpu::Specific(i); + if target & flags == flags { + let mpidr = i; + let cpu_id = MpidrValue::try_from(mpidr)?.into(); + return Ok(SpiDestination::Specific(cpu_id)); } } - let list = TargetList::from_bits_truncate(flags as u8); - TargetCpu::GICv2TargetList(list) + Ok(SpiDestination::GICv2TargetList(TargetList(flags as u8)).canonicalize()) } else if let Self::V3(v3) = self { let reg = v3.dist_extended.read_volatile_64(offset::P6IROUTER); @@ -167,11 +164,11 @@ impl super::ArmGic { // value of 1 to target any available cpu // value of 0 to target a specific cpu if reg & P6IROUTER_ANY_AVAILABLE_PE > 0 { - TargetCpu::AnyCpuAvailable + Ok(SpiDestination::AnyCpuAvailable) } else { - let aff3 = (reg >> 8) & 0xff000000; - let aff012 = reg & 0xffffff; - TargetCpu::Specific((aff3 | aff012) as u32) + let mpidr = reg & 0xff00ffffff; + let cpu_id = MpidrValue::try_from(mpidr)?.into(); + return Ok(SpiDestination::Specific(cpu_id)); } } else { // If we're on gicv2 then affinity routing is off @@ -180,40 +177,33 @@ impl super::ArmGic { } } - /// The GIC can be configured to route - /// Shared-Peripheral Interrupts (SPI) either - /// to a specific CPU or to any PE that is ready - /// to process them, i.e. not handling another - /// higher-priority interrupt. - pub fn set_spi_target(&mut self, int: InterruptNumber, target: TargetCpu) { + /// Sets the destination of an SPI. + pub fn set_spi_target(&mut self, int: InterruptNumber, target: SpiDestination) { assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU"); if !self.affinity_routing() { - assert_cpu_bounds(&target); + if let SpiDestination::Specific(cpu) = &target { + assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); + } let value = match target { - TargetCpu::Specific(cpu) => 1 << cpu, - TargetCpu::AnyCpuAvailable => 0xff, - TargetCpu::GICv2TargetList(list) => list.bits as u32, + SpiDestination::Specific(cpu) => 1 << cpu.value(), + SpiDestination::AnyCpuAvailable => { + let list = TargetList::new_all_cpus() + .expect("This is invalid: CpuId > 8 AND GICv2 interrupt controller"); + list.0 as u32 + }, + SpiDestination::GICv2TargetList(list) => list.0 as u32, }; self.distributor_mut().write_array_volatile::<4>(offset::ITARGETSR, int, value); } else if let Self::V3(v3) = self { let value = match target { - TargetCpu::Specific(cpu) => { - let cpu = cpu as u64; - // shift aff3 8 bits to the left - let aff3 = (cpu & 0xff000000) << 8; - // keep aff 0, 1 & 2 where they are - let aff012 = cpu & 0xffffff; - // leave bit 31 clear to indicate - // a specific target - aff3 | aff012 - }, + SpiDestination::Specific(cpu) => MpidrValue::from(cpu).value(), // bit 31: Interrupt Routing Mode // value of 1 to target any available cpu - TargetCpu::AnyCpuAvailable => P6IROUTER_ANY_AVAILABLE_PE, - TargetCpu::GICv2TargetList(_) => { - panic!("Cannot use TargetCpu::GICv2TargetList with GICv3!"); + SpiDestination::AnyCpuAvailable => P6IROUTER_ANY_AVAILABLE_PE, + SpiDestination::GICv2TargetList(_) => { + panic!("Cannot use SpiDestination::GICv2TargetList with GICv3!"); }, }; diff --git a/kernel/gic/src/gic/mod.rs b/kernel/gic/src/gic/mod.rs index f70fba2e3..f16493800 100644 --- a/kernel/gic/src/gic/mod.rs +++ b/kernel/gic/src/gic/mod.rs @@ -1,13 +1,13 @@ use core::convert::AsMut; -use cpu::CpuId; +use cpu::{CpuId, MpidrValue}; +use arm_boards::{BOARD_CONFIG, NUM_CPUS}; use memory::{ PageTable, BorrowedMappedPages, Mutable, PhysicalAddress, PteFlags, allocate_pages, allocate_frames_at }; use static_assertions::const_assert_eq; -use bitflags::bitflags; mod cpu_interface_gicv3; mod cpu_interface_gicv2; @@ -23,38 +23,97 @@ pub type InterruptNumber = u32; /// 8-bit unsigned integer pub type Priority = u8; -bitflags! { - /// The legacy (GICv2) way of specifying - /// multiple target CPUs - pub struct TargetList: u8 { - const CPU_0 = 1 << 0; - const CPU_1 = 1 << 1; - const CPU_2 = 1 << 2; - const CPU_3 = 1 << 3; - const CPU_4 = 1 << 4; - const CPU_5 = 1 << 5; - const CPU_6 = 1 << 6; - const CPU_7 = 1 << 7; - const ALL_CPUS = u8::MAX; +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct TargetList(u8); + +impl TargetList { + pub fn new>(list: T) -> Result { + let mut this = 0; + + for mpidr in list { + let mpidr = mpidr.value(); + if mpidr < 8 { + this |= 1 << mpidr; + } else { + return Err("CPU id is too big for GICv2 (should be < 8)"); + } + } + + Ok(Self(this)) + } + + /// Tries to create a `TargetList` from `arm_boards::BOARD_CONFIG.cpu_ids` + pub fn new_all_cpus() -> Result { + let list = BOARD_CONFIG.cpu_ids.iter().map(|def_mpidr| (*def_mpidr).into()); + Self::new(list).map_err(|_| "Some CPUs in the system cannot be stored in a TargetList") + } + + pub fn iter(self) -> TargetListIterator { + TargetListIterator { + bitfield: self.0, + shift: 0, + } } } -pub enum TargetCpu { - /// That interrupt must be handled by - /// a specific PE in the system. - /// - /// - level 3 affinity is expected in bits [24:31] - /// - level 2 affinity is expected in bits [16:23] - /// - level 1 affinity is expected in bits [8:15] - /// - level 0 affinity is expected in bits [0:7] - Specific(u32), - /// That interrupt can be handled by - /// any PE that is not busy with another, - /// more important task +pub struct TargetListIterator { + bitfield: u8, + shift: u8, +} + +impl Iterator for TargetListIterator { + type Item = Result; + fn next(&mut self) -> Option { + while ((self.bitfield >> self.shift) & 1 == 0) && self.shift < 8 { + self.shift += 1; + } + + if self.shift < 8 { + let def_mpidr = MpidrValue::try_from(self.shift as u64); + self.shift += 1; + Some(def_mpidr.map(|m| m.into())) + } else { + None + } + } +} + +/// Target of a shared-peripheral interrupt +pub enum SpiDestination { + /// The interrupt must be delivered to a specific CPU. + Specific(CpuId), + /// That interrupt can be handled by any PE that is not busy with another, more + /// important task AnyCpuAvailable, + /// The interrupt will be delivered to all CPUs specified by the included target list + GICv2TargetList(TargetList), +} + +/// Target of an inter-processor interrupt +pub enum IpiTargetCpu { + /// The interrupt will be delivered to a specific CPU. + Specific(CpuId), + /// The interrupt will be delivered to all CPUs except the sender. + AllOtherCpus, + /// The interrupt will be delivered to all CPUs specified by the included target list GICv2TargetList(TargetList), } +impl SpiDestination { + /// When this is a GICv2TargetList, converts the list to + /// `AnyCpuAvailable` if the list contains all CPUs. + pub fn canonicalize(self) -> Self { + match self { + Self::Specific(cpu_id) => Self::Specific(cpu_id), + Self::AnyCpuAvailable => Self::AnyCpuAvailable, + Self::GICv2TargetList(list) => match TargetList::new_all_cpus() == Ok(list) { + true => Self::AnyCpuAvailable, + false => Self::GICv2TargetList(list), + }, + } + } +} + const U32BITS: usize = u32::BITS as usize; #[derive(Copy, Clone)] @@ -156,7 +215,7 @@ const_assert_eq!(core::mem::size_of::(), 0x1000); /// This is defined in `arm_boards::INTERRUPT_CONTROLLER_CONFIG`. fn get_current_cpu_redist_index() -> usize { let cpu_id = cpu::current_cpu(); - arm_boards::BOARD_CONFIG.cpu_ids.iter() + BOARD_CONFIG.cpu_ids.iter() .position(|mpidr| CpuId::from(*mpidr) == cpu_id) .expect("BUG: get_current_cpu_redist_index: unexpected CpuId for current CPU") } @@ -178,7 +237,7 @@ pub struct ArmGicV3 { pub affinity_routing: Enabled, pub distributor: BorrowedMappedPages, pub dist_extended: BorrowedMappedPages, - pub redistributors: [ArmGicV3RedistPages; arm_boards::NUM_CPUS], + pub redistributors: [ArmGicV3RedistPages; NUM_CPUS], } /// Arm Generic Interrupt Controller @@ -198,7 +257,7 @@ pub enum Version { }, InitV3 { dist: PhysicalAddress, - redist: [PhysicalAddress; arm_boards::NUM_CPUS], + redist: [PhysicalAddress; NUM_CPUS], } } @@ -241,7 +300,7 @@ impl ArmGic { mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? }; - let redistributors: [ArmGicV3RedistPages; arm_boards::NUM_CPUS] = core::array::try_from_fn(|i| { + let redistributors: [ArmGicV3RedistPages; NUM_CPUS] = core::array::try_from_fn(|i| { let phys_addr = redist[i]; let mut redistributor: BorrowedMappedPages = { @@ -288,7 +347,7 @@ impl ArmGic { /// also called software generated interrupt (SGI). /// /// note: on Aarch64, IPIs must have a number below 16 on ARMv8 - pub fn send_ipi(&mut self, int_num: InterruptNumber, target: TargetCpu) { + pub fn send_ipi(&mut self, int_num: InterruptNumber, target: IpiTargetCpu) { assert!(int_num < 16, "IPIs must have a number below 16 on ARMv8"); match self {