diff --git a/Cargo.lock b/Cargo.lock index 36d5ac9f5..1020a0731 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,15 @@ dependencies = [ "tty", ] +[[package]] +name = "arm_boards" +version = "0.1.0" +dependencies = [ + "cfg-if 1.0.0", + "derive_more", + "memory_structs", +] + [[package]] name = "arrayvec" version = "0.5.2" @@ -608,6 +617,7 @@ name = "cpu" version = "0.1.0" dependencies = [ "apic", + "arm_boards", "cortex-a", "derive_more", "irq_safety", @@ -1335,7 +1345,9 @@ dependencies = [ name = "gic" version = "0.1.0" dependencies = [ + "arm_boards", "bitflags", + "cpu", "log", "memory", "static_assertions", @@ -1527,6 +1539,7 @@ name = "interrupts" version = "0.1.0" dependencies = [ "apic", + "arm_boards", "cortex-a", "cpu", "early_printer", @@ -2184,6 +2197,7 @@ dependencies = [ "acpi", "ap_start", "apic", + "arm_boards", "cpu", "kernel_config", "log", diff --git a/kernel/arm_boards/Cargo.toml b/kernel/arm_boards/Cargo.toml new file mode 100644 index 000000000..12e3aee82 --- /dev/null +++ b/kernel/arm_boards/Cargo.toml @@ -0,0 +1,18 @@ +[package] +authors = ["Nathan Royer "] +name = "arm_boards" +description = "Board-specific configuration and definitions for aarch64 systems" +version = "0.1.0" +edition = "2021" + +[dependencies] +cfg-if = "1.0.0" +memory_structs = { path = "../memory_structs" } +derive_more = "0.99.0" + +[features] +default = [ "qemu_virt" ] +qemu_virt = [] + +[lib] +crate-type = ["rlib"] diff --git a/kernel/arm_boards/src/boards/qemu_virt.rs b/kernel/arm_boards/src/boards/qemu_virt.rs new file mode 100644 index 000000000..f79ddcfed --- /dev/null +++ b/kernel/arm_boards/src/boards/qemu_virt.rs @@ -0,0 +1,36 @@ +//! Board configuration for QEMU's basic `virt` machine with 4 CPUs. + +use super::{ + InterruptControllerConfig::GicV3, GicV3InterruptControllerConfig, + BoardConfig, mpidr::DefinedMpidrValue, +}; +use memory_structs::PhysicalAddress; + +/// Generates an MPIDR value from a CPU's 0th affinity level. +const fn cpu_id(aff0: u8) -> DefinedMpidrValue { + DefinedMpidrValue::new(0, 0, 0, aff0) +} + +/// Generates a Redistributor base address from a CPU's 0th affinity level. +const fn redist(aff0: usize) -> PhysicalAddress { + PhysicalAddress::new_canonical(0x080A0000 + 0x20000 * aff0) +} + +pub const NUM_CPUS: usize = 4; +pub static BOARD_CONFIG: BoardConfig = BoardConfig { + cpu_ids: [ + cpu_id(0), + cpu_id(1), + cpu_id(2), + cpu_id(3), + ], + interrupt_controller: GicV3(GicV3InterruptControllerConfig { + distributor_base_address: PhysicalAddress::new_canonical(0x08000000), + redistributor_base_addresses: [ + redist(0), + redist(1), + redist(2), + redist(3), + ], + }), +}; diff --git a/kernel/arm_boards/src/boards/unselected.rs b/kernel/arm_boards/src/boards/unselected.rs new file mode 100644 index 000000000..77e79ddd1 --- /dev/null +++ b/kernel/arm_boards/src/boards/unselected.rs @@ -0,0 +1,3 @@ +//! Invalid board config file that throws a compile error if a board config wasn't selected. + +compile_error!("Please select a board config feature in the arm_boards crate"); diff --git a/kernel/arm_boards/src/lib.rs b/kernel/arm_boards/src/lib.rs new file mode 100644 index 000000000..4ea1dc20e --- /dev/null +++ b/kernel/arm_boards/src/lib.rs @@ -0,0 +1,53 @@ +//! Configuration and definitions for specific boards on aarch64 systems. +//! +//! | Board Name | Num CPUs | Interrupt Controller | Secondary CPU Startup Method | +//! | ---------- | --------- | -------------------- | ---------------------------- | +//! | qemu_virt | 4 | GICv3 | PSCI | +//! + +#![no_std] +#![feature(const_trait_impl)] + +cfg_if::cfg_if! { +if #[cfg(target_arch = "aarch64")] { + +use memory_structs::PhysicalAddress; + +#[derive(Debug, Copy, Clone)] +pub struct GicV3InterruptControllerConfig { + pub distributor_base_address: PhysicalAddress, + pub redistributor_base_addresses: [PhysicalAddress; board::NUM_CPUS], +} + +#[derive(Debug, Copy, Clone)] +pub enum InterruptControllerConfig { + GicV3(GicV3InterruptControllerConfig), +} + +/* +TODO: multicore_bringup: wake secondary cores based on this: +pub enum SecondaryCoresStartup { + Psci, +} +*/ + +#[derive(Debug, Clone)] +pub struct BoardConfig { + pub cpu_ids: [mpidr::DefinedMpidrValue; board::NUM_CPUS], + pub interrupt_controller: InterruptControllerConfig, +} + +// by default & on x86_64, the default.rs file is used +#[cfg_attr(feature = "qemu_virt", path = "boards/qemu_virt.rs")] +#[cfg_attr(not(any( + feature = "qemu_virt", +)), path = "boards/unselected.rs")] +mod board; + +pub mod mpidr; + +pub use board::{NUM_CPUS, BOARD_CONFIG}; + + +} // end of cfg(target_arch = "aarch64") +} // end of cfg_if diff --git a/kernel/arm_boards/src/mpidr.rs b/kernel/arm_boards/src/mpidr.rs new file mode 100644 index 000000000..958b1a67a --- /dev/null +++ b/kernel/arm_boards/src/mpidr.rs @@ -0,0 +1,25 @@ +use derive_more::{Display, Binary, Octal, LowerHex, UpperHex}; + +/// A unique identifier for a CPU, hardcoded in `arm_boards`. +#[derive( + Clone, Copy, Debug, Display, PartialEq, Eq, PartialOrd, Ord, + Hash, Binary, Octal, LowerHex, UpperHex, +)] +#[repr(transparent)] +pub struct DefinedMpidrValue(u64); + +impl DefinedMpidrValue { + /// Returns the contained value + pub fn value(self) -> u64 { + self.0 + } + + /// Create an `MpidrValue` from its four affinity numbers + pub(crate) const fn new(aff3: u8, aff2: u8, aff1: u8, aff0: u8) -> Self { + let aff3 = (aff3 as u64) << 32; + let aff2 = (aff2 as u64) << 16; + let aff1 = (aff1 as u64) << 8; + let aff0 = (aff0 as u64) << 0; + Self(aff3 | aff2 | aff1 | aff0) + } +} diff --git a/kernel/cpu/Cargo.toml b/kernel/cpu/Cargo.toml index 275ed2279..2348e2aa2 100644 --- a/kernel/cpu/Cargo.toml +++ b/kernel/cpu/Cargo.toml @@ -8,14 +8,14 @@ edition = "2021" [dependencies] derive_more = "0.99.0" - [target.'cfg(target_arch = "x86_64")'.dependencies] apic = { path = "../apic" } [target.'cfg(target_arch = "aarch64")'.dependencies] +irq_safety = { git = "https://github.com/theseus-os/irq_safety" } +arm_boards = { path = "../arm_boards" } tock-registers = "0.7.0" cortex-a = "7.5.0" -irq_safety = { git = "https://github.com/theseus-os/irq_safety" } [lib] crate-type = ["rlib"] diff --git a/kernel/cpu/src/aarch64.rs b/kernel/cpu/src/aarch64.rs index 2cb3f10d5..3fe5128a9 100644 --- a/kernel/cpu/src/aarch64.rs +++ b/kernel/cpu/src/aarch64.rs @@ -6,6 +6,7 @@ use derive_more::{Display, Binary, Octal, LowerHex, UpperHex}; use irq_safety::RwLockIrqSafe; use core::fmt; use alloc::vec::Vec; +use arm_boards::mpidr::DefinedMpidrValue; use super::CpuId; @@ -89,14 +90,17 @@ impl MpidrValue { }; (self.0 >> shift) as u8 } +} + +impl From for MpidrValue { + fn from(def_mpidr: DefinedMpidrValue) -> Self { + Self(def_mpidr.value()) + } +} - /// Create an `MpidrValue` from its four affinity numbers - pub fn new(aff3: u8, aff2: u8, aff1: u8, aff0: u8) -> Self { - let aff3 = (aff3 as u64) << 32; - let aff2 = (aff2 as u64) << 16; - let aff1 = (aff1 as u64) << 8; - let aff0 = (aff0 as u64) << 0; - Self(aff3 | aff2 | aff1 | aff0) +impl From for CpuId { + fn from(def_mpidr: DefinedMpidrValue) -> Self { + Self::from(MpidrValue::from(def_mpidr)) } } diff --git a/kernel/gic/Cargo.toml b/kernel/gic/Cargo.toml index b524c862e..8a5e5202d 100644 --- a/kernel/gic/Cargo.toml +++ b/kernel/gic/Cargo.toml @@ -12,3 +12,5 @@ zerocopy = "0.5.0" log = "0.4.8" memory = { path = "../memory" } +cpu = { path = "../cpu" } +arm_boards = { path = "../arm_boards" } diff --git a/kernel/gic/src/gic/cpu_interface_gicv2.rs b/kernel/gic/src/gic/cpu_interface_gicv2.rs index 65049b635..264297e5f 100644 --- a/kernel/gic/src/gic/cpu_interface_gicv2.rs +++ b/kernel/gic/src/gic/cpu_interface_gicv2.rs @@ -7,17 +7,16 @@ //! - Sending End-Of-Interrupts signals use super::GicRegisters; -use super::U32BYTES; use super::Priority; use super::InterruptNumber; mod offset { - use super::U32BYTES; - pub const CTLR: usize = 0x00 / U32BYTES; - pub const PMR: usize = 0x04 / U32BYTES; - pub const IAR: usize = 0x0C / U32BYTES; - pub const RPR: usize = 0x14 / U32BYTES; - pub const EOIR: usize = 0x10 / U32BYTES; + use crate::Offset32; + pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00); + pub(crate) const PMR: Offset32 = Offset32::from_byte_offset(0x04); + pub(crate) const IAR: Offset32 = Offset32::from_byte_offset(0x0C); + pub(crate) const RPR: Offset32 = Offset32::from_byte_offset(0x14); + pub(crate) const EOIR: Offset32 = Offset32::from_byte_offset(0x10); } // enable group 0 diff --git a/kernel/gic/src/gic/dist_interface.rs b/kernel/gic/src/gic/dist_interface.rs index 312b3e757..158d72e51 100644 --- a/kernel/gic/src/gic/dist_interface.rs +++ b/kernel/gic/src/gic/dist_interface.rs @@ -13,22 +13,21 @@ //! - Generating software interrupts (GICv2 style) use super::GicRegisters; -use super::U32BYTES; use super::TargetCpu; use super::InterruptNumber; use super::Enabled; use super::TargetList; mod offset { - use super::U32BYTES; - pub const CTLR: usize = 0x000 / U32BYTES; - pub const IGROUPR: usize = 0x080 / U32BYTES; - pub const ISENABLER: usize = 0x100 / U32BYTES; - pub const ICENABLER: usize = 0x180 / U32BYTES; - pub const ITARGETSR: usize = 0x800 / U32BYTES; - pub const SGIR: usize = 0xf00 / U32BYTES; + use crate::{Offset32, Offset64}; + pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x000); + pub(crate) const IGROUPR: Offset32 = Offset32::from_byte_offset(0x080); + pub(crate) const ISENABLER: Offset32 = Offset32::from_byte_offset(0x100); + pub(crate) const ICENABLER: Offset32 = Offset32::from_byte_offset(0x180); + pub(crate) const ITARGETSR: Offset32 = Offset32::from_byte_offset(0x800); + pub(crate) const SGIR: Offset32 = Offset32::from_byte_offset(0xf00); /// This one is on the 6th page - pub const P6IROUTER: usize = 0x100 / U32BYTES; + pub(crate) const P6IROUTER: Offset64 = Offset64::from_byte_offset(0x100); } // enable group 0 @@ -37,6 +36,9 @@ mod offset { // enable group 1 const CTLR_ENGRP1: u32 = 0b10; +// enable 1 of N wakeup functionality +const CTLR_E1NWF: u32 = 1 << 7; + // Affinity Routing Enable, Non-secure state. const CTLR_ARE_NS: u32 = 1 << 5; @@ -48,7 +50,7 @@ const SGIR_TARGET_ALL_OTHER_PE: u32 = 1 << 24; // bit 31: SPI routing // 1 = any available PE // 0 = route to specific PE -const P6IROUTER_ANY_AVAILABLE_PE: u32 = 1 << 31; +const P6IROUTER_ANY_AVAILABLE_PE: u64 = 1 << 31; // const GROUP_0: u32 = 0; const GROUP_1: u32 = 1; @@ -63,7 +65,8 @@ fn assert_cpu_bounds(target: &TargetCpu) { } /// Initializes the distributor by enabling forwarding -/// of group 1 interrupts +/// of group 1 interrupts and allowing the GIC to pick +/// a core that is asleep for "1 of N" interrupts. /// /// Return value: whether or not affinity routing is /// currently enabled for both secure and non-secure @@ -71,6 +74,7 @@ fn assert_cpu_bounds(target: &TargetCpu) { pub fn init(registers: &mut GicRegisters) -> Enabled { let mut reg = registers.read_volatile(offset::CTLR); reg |= CTLR_ENGRP1; + reg |= CTLR_E1NWF; registers.write_volatile(offset::CTLR, reg); // Return value: whether or not affinity routing is @@ -157,7 +161,7 @@ impl super::ArmGic { let list = TargetList::from_bits_truncate(flags as u8); TargetCpu::GICv2TargetList(list) } else if let Self::V3(v3) = self { - let reg = v3.dist_extended.read_volatile(offset::P6IROUTER); + let reg = v3.dist_extended.read_volatile_64(offset::P6IROUTER); // bit 31: Interrupt Routing Mode // value of 1 to target any available cpu @@ -167,7 +171,7 @@ impl super::ArmGic { } else { let aff3 = (reg >> 8) & 0xff000000; let aff012 = reg & 0xffffff; - TargetCpu::Specific(aff3 | aff012) + TargetCpu::Specific((aff3 | aff012) as u32) } } else { // If we're on gicv2 then affinity routing is off @@ -196,6 +200,7 @@ impl super::ArmGic { } else if let Self::V3(v3) = self { let value = match target { TargetCpu::Specific(cpu) => { + let cpu = cpu as u64; // shift aff3 8 bits to the left let aff3 = (cpu & 0xff000000) << 8; // keep aff 0, 1 & 2 where they are @@ -212,7 +217,7 @@ impl super::ArmGic { }, }; - v3.dist_extended.write_volatile(offset::P6IROUTER, value); + v3.dist_extended.write_volatile_64(offset::P6IROUTER, value); } // If we're on gicv2 then affinity routing is off diff --git a/kernel/gic/src/gic/mod.rs b/kernel/gic/src/gic/mod.rs index 301e6b4d9..f70fba2e3 100644 --- a/kernel/gic/src/gic/mod.rs +++ b/kernel/gic/src/gic/mod.rs @@ -1,7 +1,10 @@ use core::convert::AsMut; -use memory::{PageTable, BorrowedMappedPages, Mutable, -PhysicalAddress, PteFlags, allocate_pages, allocate_frames_at}; +use cpu::CpuId; +use memory::{ + PageTable, BorrowedMappedPages, Mutable, PhysicalAddress, PteFlags, + allocate_pages, allocate_frames_at +}; use static_assertions::const_assert_eq; use bitflags::bitflags; @@ -11,16 +14,6 @@ mod cpu_interface_gicv2; mod dist_interface; mod redist_interface; -/// Physical addresses of the CPU and Distributor -/// interfaces as exposed by the qemu "virt" VM. -pub mod qemu_virt_addrs { - use super::*; - - pub const GICD: PhysicalAddress = PhysicalAddress::new_canonical(0x08000000); - pub const GICC: PhysicalAddress = PhysicalAddress::new_canonical(0x08010000); - pub const GICR: PhysicalAddress = PhysicalAddress::new_canonical(0x080A0000); -} - /// Boolean pub type Enabled = bool; @@ -62,9 +55,26 @@ pub enum TargetCpu { GICv2TargetList(TargetList), } -const U32BYTES: usize = core::mem::size_of::(); const U32BITS: usize = u32::BITS as usize; +#[derive(Copy, Clone)] +pub(crate) struct Offset32(usize); + +#[derive(Copy, Clone)] +pub(crate) struct Offset64(usize); + +impl Offset32 { + pub(crate) const fn from_byte_offset(byte_offset: usize) -> Self { + Self(byte_offset / core::mem::size_of::()) + } +} + +impl Offset64 { + pub(crate) const fn from_byte_offset(byte_offset: usize) -> Self { + Self(byte_offset / core::mem::size_of::()) + } +} + #[repr(C)] #[derive(zerocopy::FromBytes)] pub struct GicRegisters { @@ -72,12 +82,20 @@ pub struct GicRegisters { } impl GicRegisters { - fn read_volatile(&self, index: usize) -> u32 { - unsafe { (&self.inner[index] as *const u32).read_volatile() } + fn read_volatile(&self, offset: Offset32) -> u32 { + unsafe { (&self.inner[offset.0] as *const u32).read_volatile() } + } + + fn write_volatile(&mut self, offset: Offset32, value: u32) { + unsafe { (&mut self.inner[offset.0] as *mut u32).write_volatile(value) } + } + + fn read_volatile_64(&self, offset: Offset64) -> u64 { + unsafe { (self.inner.as_ptr() as *const u64).add(offset.0).read_volatile() } } - fn write_volatile(&mut self, index: usize, value: u32) { - unsafe { (&mut self.inner[index] as *mut u32).write_volatile(value) } + fn write_volatile_64(&mut self, offset: Offset64, value: u64) { + unsafe { (self.inner.as_mut_ptr() as *mut u64).add(offset.0).write_volatile(value) } } // Reads one item of an array spanning across @@ -90,12 +108,12 @@ impl GicRegisters { // - `int` is the index // - `offset` tells the beginning of the array // - `INTS_PER_U32` = how many array slots per u32 in this array - fn read_array_volatile(&self, offset: usize, int: InterruptNumber) -> u32 { + fn read_array_volatile(&self, offset: Offset32, int: InterruptNumber) -> u32 { let int = int as usize; let bits_per_int: usize = U32BITS / INTS_PER_U32; let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); - let offset = offset + (int / INTS_PER_U32); + let offset = Offset32(offset.0 + (int / INTS_PER_U32)); let reg_index = int & (INTS_PER_U32 - 1); let shift = reg_index * bits_per_int; @@ -114,12 +132,12 @@ impl GicRegisters { // - `offset` tells the beginning of the array // - `INTS_PER_U32` = how many array slots per u32 in this array // - `value` is the value to write - fn write_array_volatile(&mut self, offset: usize, int: InterruptNumber, value: u32) { + fn write_array_volatile(&mut self, offset: Offset32, int: InterruptNumber, value: u32) { let int = int as usize; let bits_per_int: usize = U32BITS / INTS_PER_U32; let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); - let offset = offset + (int / INTS_PER_U32); + let offset = Offset32(offset.0 + (int / INTS_PER_U32)); let reg_index = int & (INTS_PER_U32 - 1); let shift = reg_index * bits_per_int; @@ -132,6 +150,17 @@ impl GicRegisters { const_assert_eq!(core::mem::size_of::(), 0x1000); +/// Returns the index to the redistributor base address for this CPU +/// in the array of register base addresses. +/// +/// This is defined in `arm_boards::INTERRUPT_CONTROLLER_CONFIG`. +fn get_current_cpu_redist_index() -> usize { + let cpu_id = cpu::current_cpu(); + arm_boards::BOARD_CONFIG.cpu_ids.iter() + .position(|mpidr| CpuId::from(*mpidr) == cpu_id) + .expect("BUG: get_current_cpu_redist_index: unexpected CpuId for current CPU") +} + const REDIST_SGIPPI_OFFSET: usize = 0x10000; const DIST_P6_OFFSET: usize = 0x6000; @@ -140,12 +169,16 @@ pub struct ArmGicV2 { pub processor: BorrowedMappedPages, } +pub struct ArmGicV3RedistPages { + pub redistributor: BorrowedMappedPages, + pub redist_sgippi: BorrowedMappedPages, +} + pub struct ArmGicV3 { pub affinity_routing: Enabled, pub distributor: BorrowedMappedPages, pub dist_extended: BorrowedMappedPages, - pub redistributor: BorrowedMappedPages, - pub redist_sgippi: BorrowedMappedPages, + pub redistributors: [ArmGicV3RedistPages; arm_boards::NUM_CPUS], } /// Arm Generic Interrupt Controller @@ -165,7 +198,7 @@ pub enum Version { }, InitV3 { dist: PhysicalAddress, - redist: PhysicalAddress, + redist: [PhysicalAddress; arm_boards::NUM_CPUS], } } @@ -208,25 +241,38 @@ impl ArmGic { mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? }; - let mut redistributor: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the redistributor interface")?; - let frames = allocate_frames_at(redist, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, mmio_flags)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? - }; + let redistributors: [ArmGicV3RedistPages; arm_boards::NUM_CPUS] = core::array::try_from_fn(|i| { + let phys_addr = redist[i]; - let redist_sgippi = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the extended redistributor interface")?; - let frames = allocate_frames_at(redist + REDIST_SGIPPI_OFFSET, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, mmio_flags)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? - }; + let mut redistributor: BorrowedMappedPages = { + let pages = allocate_pages(1).ok_or("couldn't allocate pages for the redistributor interface")?; + let frames = allocate_frames_at(phys_addr, 1)?; + let mapped = page_table.map_allocated_pages_to(pages, frames, mmio_flags)?; + mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? + }; + + redist_interface::init(redistributor.as_mut())?; + + let redist_sgippi = { + let pages = allocate_pages(1).ok_or("couldn't allocate pages for the extended redistributor interface")?; + let frames = allocate_frames_at(phys_addr + REDIST_SGIPPI_OFFSET, 1)?; + let mapped = page_table.map_allocated_pages_to(pages, frames, mmio_flags)?; + mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? + }; + + Ok::(ArmGicV3RedistPages { + redistributor, + redist_sgippi, + }) + })?; + + // this cannot fail as we pushed exactly `arm_boards::CPUS` items + // let redistributors = redistributors.into_inner().unwrap(); - redist_interface::init(redistributor.as_mut())?; cpu_interface_gicv3::init(); let affinity_routing = dist_interface::init(distributor.as_mut()); - Ok(Self::V3(ArmGicV3 { distributor, dist_extended, redistributor, redist_sgippi, affinity_routing })) + Ok(Self::V3(ArmGicV3 { distributor, dist_extended, redistributors, affinity_routing })) }, } } @@ -272,7 +318,8 @@ impl ArmGic { pub fn get_interrupt_state(&self, int: InterruptNumber) -> Enabled { match int { 0..=31 => if let Self::V3(v3) = self { - redist_interface::is_sgippi_enabled(&v3.redist_sgippi, int) + let i = get_current_cpu_redist_index(); + redist_interface::is_sgippi_enabled(&v3.redistributors[i].redist_sgippi, int) } else { true }, @@ -285,7 +332,8 @@ impl ArmGic { pub fn set_interrupt_state(&mut self, int: InterruptNumber, enabled: Enabled) { match int { 0..=31 => if let Self::V3(v3) = self { - redist_interface::enable_sgippi(&mut v3.redist_sgippi, int, enabled); + let i = get_current_cpu_redist_index(); + redist_interface::enable_sgippi(&mut v3.redistributors[i].redist_sgippi, int, enabled); }, _ => dist_interface::enable_spi(self.distributor_mut(), int, enabled), }; @@ -313,3 +361,9 @@ impl ArmGic { } } } + +impl core::fmt::Debug for ArmGicV3RedistPages { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "ArmGicV3RedistPages") + } +} diff --git a/kernel/gic/src/gic/redist_interface.rs b/kernel/gic/src/gic/redist_interface.rs index a5212fc7a..efe6f802b 100644 --- a/kernel/gic/src/gic/redist_interface.rs +++ b/kernel/gic/src/gic/redist_interface.rs @@ -9,69 +9,92 @@ //! - Enabling or disabling the forwarding of PPIs & SGIs based on their numbers use super::GicRegisters; -use super::U32BYTES; use super::InterruptNumber; use super::Enabled; mod offset { - use super::U32BYTES; - pub const RD_WAKER: usize = 0x14 / U32BYTES; - pub const IGROUPR: usize = 0x80 / U32BYTES; - pub const SGIPPI_ISENABLER: usize = 0x100 / U32BYTES; - pub const SGIPPI_ICENABLER: usize = 0x180 / U32BYTES; + use crate::{Offset32, Offset64}; + pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00); + pub(crate) const TYPER: Offset64 = Offset64::from_byte_offset(0x08); + pub(crate) const WAKER: Offset32 = Offset32::from_byte_offset(0x14); + pub(crate) const IGROUPR: Offset32 = Offset32::from_byte_offset(0x80); + pub(crate) const SGIPPI_ISENABLER: Offset32 = Offset32::from_byte_offset(0x100); + pub(crate) const SGIPPI_ICENABLER: Offset32 = Offset32::from_byte_offset(0x180); } -const RD_WAKER_PROCESSOR_SLEEP: u32 = 1 << 1; -const RD_WAKER_CHLIDREN_ASLEEP: u32 = 1 << 2; +const WAKER_PROCESSOR_SLEEP: u32 = 1 << 1; +const WAKER_CHLIDREN_ASLEEP: u32 = 1 << 2; -// const GROUP_0: u32 = 0; +/// Bit that is set if GICR_CTLR.DPG* bits are supported +const TYPER_DPGS: u64 = 1 << 5; + +/// If bit is set, the PE cannot be selected for non-secure group 1 "1 of N" interrupts. +const CTLR_DPG1S: u32 = 1 << 26; + +/// If bit is set, the PE cannot be selected for secure group 1 "1 of N" interrupts. +const CTLR_DPG1NS: u32 = 1 << 25; + +/// If bit is set, the PE cannot be selected for group 0 "1 of N" interrupts. +const CTLR_DPG0: u32 = 1 << 24; + +/// const GROUP_0: u32 = 0; const GROUP_1: u32 = 1; -// This timeout value works on some ARM SoCs: -// - qemu's virt virtual machine -// -// (if the value works for your SoC, please add it to this list.) -// -// If the redistributor's initialization times out, it means either: -// - that your ARM SoC is not GICv3 compliant (try initializing it as GICv2) -// - that the timeout value is too low for your ARM SoC. Try increasing it -// to see if the booting sequence continues. -// -// If it wasn't enough for your machine, reach out to the Theseus -// developers (or directly submit a PR). +/// This timeout value works on some ARM SoCs: +/// - qemu's virt virtual machine +/// +/// (if the value works for your SoC, please add it to this list.) +/// +/// If the redistributor's initialization times out, it means either: +/// - that your ARM SoC is not GICv3 compliant (try initializing it as GICv2) +/// - that the timeout value is too low for your ARM SoC. Try increasing it +/// to see if the booting sequence continues. +/// +/// If it wasn't enough for your machine, reach out to the Theseus +/// developers (or directly submit a PR). const TIMEOUT_ITERATIONS: usize = 0xffff; -/// Initializes the redistributor by waking -/// it up and checking that it's awake +/// Initializes the redistributor by waking it up and waiting for it to awaken. +/// +/// Returns an error if a timeout occurs while waiting. pub fn init(registers: &mut GicRegisters) -> Result<(), &'static str> { - let mut reg; - reg = registers.read_volatile(offset::RD_WAKER); + let mut reg = registers.read_volatile(offset::WAKER); // Wake the redistributor - reg &= !RD_WAKER_PROCESSOR_SLEEP; - - registers.write_volatile(offset::RD_WAKER, reg); - - // then poll ChildrenAsleep until it's cleared + reg &= !WAKER_PROCESSOR_SLEEP; + registers.write_volatile(offset::WAKER, reg); + // Then, wait for the children to wake up, timing out if it never happens. let children_asleep = || { - registers.read_volatile(offset::RD_WAKER) & RD_WAKER_CHLIDREN_ASLEEP > 0 + registers.read_volatile(offset::WAKER) & WAKER_CHLIDREN_ASLEEP > 0 }; - let mut counter = 0; - let mut timed_out = || { + while children_asleep() { counter += 1; - counter >= TIMEOUT_ITERATIONS - }; + if counter >= TIMEOUT_ITERATIONS { + break; + } + } + + if counter >= TIMEOUT_ITERATIONS { + return Err("BUG: gic driver: The redistributor didn't wake up in time."); + } + + if registers.read_volatile_64(offset::TYPER) & TYPER_DPGS != 0 { + // DPGS bits are supported in GICR_CTLR + let mut reg = registers.read_volatile(offset::CTLR); - while children_asleep() && !timed_out() { } + // Enable PE selection for non-secure group 1 SPIs + reg &= !CTLR_DPG1NS; - match timed_out() { - false => Ok(()), + // Disable PE selection for group 0 & secure group 1 SPIs + reg |= CTLR_DPG0; + reg |= CTLR_DPG1S; - // see definition of TIMEOUT_ITERATIONS - true => Err("gic: The redistributor didn't wake up in time."), + registers.write_volatile(offset::CTLR, reg); } + + Ok(()) } /// Returns whether the given SGI (software generated interrupts) or diff --git a/kernel/gic/src/lib.rs b/kernel/gic/src/lib.rs index 67aeef9a1..2861978bd 100644 --- a/kernel/gic/src/lib.rs +++ b/kernel/gic/src/lib.rs @@ -8,9 +8,10 @@ #![no_std] #![feature(doc_cfg)] +#![feature(array_try_from_fn)] -#[cfg(any(target_arch = "aarch64", doc))] +#[cfg(target_arch = "aarch64")] mod gic; -#[cfg(any(target_arch = "aarch64", doc))] +#[cfg(target_arch = "aarch64")] pub use gic::*; diff --git a/kernel/interrupts/Cargo.toml b/kernel/interrupts/Cargo.toml index ce3c9788d..093a5dd53 100644 --- a/kernel/interrupts/Cargo.toml +++ b/kernel/interrupts/Cargo.toml @@ -14,6 +14,7 @@ spin = "0.9.4" [target.'cfg(target_arch = "aarch64")'.dependencies] irq_safety = { git = "https://github.com/theseus-os/irq_safety" } +arm_boards = { path = "../arm_boards" } kernel_config = { path = "../kernel_config" } gic = { path = "../gic" } tock-registers = "0.7.0" diff --git a/kernel/interrupts/src/aarch64/mod.rs b/kernel/interrupts/src/aarch64/mod.rs index 2bbfb293a..852664689 100644 --- a/kernel/interrupts/src/aarch64/mod.rs +++ b/kernel/interrupts/src/aarch64/mod.rs @@ -9,7 +9,8 @@ use tock_registers::interfaces::Readable; use tock_registers::registers::InMemoryRegister; use kernel_config::time::CONFIG_TIMESLICE_PERIOD_MICROSECONDS; -use gic::{qemu_virt_addrs, ArmGic, InterruptNumber, Version as GicVersion}; +use arm_boards::{BOARD_CONFIG, InterruptControllerConfig}; +use gic::{ArmGic, InterruptNumber, Version as GicVersion}; use irq_safety::{RwLockIrqSafe, MutexIrqSafe}; use memory::get_kernel_mmi_ref; use log::{info, error}; @@ -21,7 +22,7 @@ use time::{Monotonic, ClockSource, Instant, Period, register_clock_source}; global_asm!(include_str!("table.s")); // The global Generic Interrupt Controller singleton -static GIC: MutexIrqSafe> = MutexIrqSafe::new(None); +static INTERRUPT_CONTROLLER: MutexIrqSafe> = MutexIrqSafe::new(None); /// The IRQ number reserved for CPU-local timer interrupts, /// which Theseus currently uses for preemptive task switching. @@ -124,9 +125,9 @@ pub fn init_ap() { set_vbar_el1(); // Enable the CPU-local timer - let mut gic = GIC.lock(); - let gic = gic.as_mut().expect("BUG: init_ap(): GIC was uninitialized"); - gic.set_interrupt_state(CPU_LOCAL_TIMER_IRQ, true); + let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); + let int_ctrl = int_ctrl.as_mut().expect("BUG: init_ap(): INTERRUPT_CONTROLLER was uninitialized"); + int_ctrl.set_interrupt_state(CPU_LOCAL_TIMER_IRQ, true); enable_timer(true); } @@ -139,38 +140,42 @@ pub fn init() -> Result<(), &'static str> { let period_femtoseconds = read_timer_period_femtoseconds(); register_clock_source::(Period::new(period_femtoseconds)); - let mut gic = GIC.lock(); - if gic.is_some() { - Err("The GIC has already been initialized!") + let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); + if int_ctrl.is_some() { + Err("The interrupt controller has already been initialized!") } else { set_vbar_el1(); - let kernel_mmi_ref = get_kernel_mmi_ref() - .ok_or("logger_aarch64: couldn't get kernel MMI ref")?; - - let mut mmi = kernel_mmi_ref.lock(); - let page_table = &mut mmi.deref_mut().page_table; - - info!("Configuring the GIC"); - let mut inner = ArmGic::init( - page_table, - GicVersion::InitV3 { - dist: qemu_virt_addrs::GICD, - redist: qemu_virt_addrs::GICR, + info!("Configuring the interrupt controller"); + match BOARD_CONFIG.interrupt_controller { + InterruptControllerConfig::GicV3(gicv3_cfg) => { + let kernel_mmi_ref = get_kernel_mmi_ref() + .ok_or("logger_aarch64: couldn't get kernel MMI ref")?; + + let mut mmi = kernel_mmi_ref.lock(); + let page_table = &mut mmi.deref_mut().page_table; + + let mut inner = ArmGic::init( + page_table, + GicVersion::InitV3 { + dist: gicv3_cfg.distributor_base_address, + redist: gicv3_cfg.redistributor_base_addresses, + }, + )?; + + inner.set_minimum_priority(0); + *int_ctrl = Some(inner); }, - )?; - - inner.set_minimum_priority(0); - *gic = Some(inner); + } - info!("Done Configuring the GIC"); + info!("Done Configuring the interrupt controller"); Ok(()) } } /// This function registers an interrupt handler for the CPU-local -/// timer and handles GIC configuration for the timer interrupt. +/// timer and handles INTERRUPT_CONTROLLER configuration for the timer interrupt. pub fn init_timer(timer_tick_handler: HandlerFunc) -> Result<(), &'static str> { // register/deregister the handler for the timer IRQ. if let Err(existing_handler) = register_interrupt(CPU_LOCAL_TIMER_IRQ, timer_tick_handler) { @@ -181,11 +186,11 @@ pub fn init_timer(timer_tick_handler: HandlerFunc) -> Result<(), &'static str> { // Route the IRQ to this core (implicit as IRQ < 32) & Enable the interrupt. { - let mut gic = GIC.lock(); - let gic = gic.as_mut().ok_or("GIC is uninitialized")?; + let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); + let int_ctrl = int_ctrl.as_mut().ok_or("INTERRUPT_CONTROLLER is uninitialized")?; // enable routing of this interrupt - gic.set_interrupt_state(CPU_LOCAL_TIMER_IRQ, true); + int_ctrl.set_interrupt_state(CPU_LOCAL_TIMER_IRQ, true); } Ok(()) @@ -273,9 +278,9 @@ pub fn deregister_interrupt(irq_num: InterruptNumber, func: HandlerFunc) -> Resu /// Send an "end of interrupt" signal, notifying the interrupt chip that /// the given interrupt request `irq` has been serviced. pub fn eoi(irq_num: InterruptNumber) { - let mut gic = GIC.lock(); - let gic = gic.as_mut().expect("BUG: eoi(): GIC was uninitialized"); - gic.end_of_interrupt(irq_num); + let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); + let int_ctrl = int_ctrl.as_mut().expect("BUG: eoi(): INTERRUPT_CONTROLLER was uninitialized"); + int_ctrl.end_of_interrupt(irq_num); } // A ClockSource for the time crate, implemented using @@ -390,13 +395,13 @@ extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { extern "C" fn current_elx_irq(exc: &mut ExceptionContext) { // read IRQ num // read IRQ priority - // ackownledge IRQ to the GIC + // ackownledge IRQ to the INTERRUPT_CONTROLLER let (irq_num, _priority) = { - let mut gic = GIC.lock(); - let gic = gic.as_mut().expect("GIC is uninitialized"); - gic.acknowledge_interrupt() + let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); + let int_ctrl = int_ctrl.as_mut().expect("INTERRUPT_CONTROLLER is uninitialized"); + int_ctrl.acknowledge_interrupt() }; - // important: GIC mutex is now implicitly unlocked + // important: INTERRUPT_CONTROLLER mutex is now implicitly unlocked let irq_num_usize = irq_num as usize; let handler = match irq_num_usize < MAX_IRQ_NUM { @@ -406,8 +411,8 @@ extern "C" fn current_elx_irq(exc: &mut ExceptionContext) { if handler(exc) == EoiBehaviour::CallerMustSignalEoi { // handler has returned, we can lock again - let mut gic = GIC.lock(); - gic.as_mut().unwrap().end_of_interrupt(irq_num); + let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); + int_ctrl.as_mut().unwrap().end_of_interrupt(irq_num); } } diff --git a/kernel/multicore_bringup/Cargo.toml b/kernel/multicore_bringup/Cargo.toml index 43792ac50..723cebb88 100644 --- a/kernel/multicore_bringup/Cargo.toml +++ b/kernel/multicore_bringup/Cargo.toml @@ -19,6 +19,7 @@ kernel_config = { path = "../kernel_config" } [target.'cfg(target_arch = "aarch64")'.dependencies] psci = "0.1.1" memory_aarch64 = { path = "../memory_aarch64" } +arm_boards = { path = "../arm_boards" } [target.'cfg(target_arch = "x86_64")'.dependencies] pit_clock_basic = { path = "../pit_clock_basic" } diff --git a/kernel/multicore_bringup/src/aarch64.rs b/kernel/multicore_bringup/src/aarch64.rs index 82cfc4f4c..be4d594d1 100644 --- a/kernel/multicore_bringup/src/aarch64.rs +++ b/kernel/multicore_bringup/src/aarch64.rs @@ -7,6 +7,7 @@ use ap_start::kstart_ap; use volatile::Volatile; use core::arch::asm; use cpu::{CpuId, MpidrValue, current_cpu}; +use arm_boards::BOARD_CONFIG; /// The data items used when an AP core is booting up in ap_entry_point & ap_stage_two. #[cfg(target_arch = "aarch64")] @@ -93,68 +94,60 @@ pub fn handle_ap_cores( } let mut ap_stack = None; - log::error!("Bruteforcing 256 CpuIds that are likely to be valid"); - for aff1 in 0..16 { - for aff0 in 0..16 { - let mpidr = MpidrValue::new(0, 0, aff1, aff0); - let cpu_id: CpuId = mpidr.into(); - - ap_data.ap_ready.write(0); - let stack = if let Some(stack) = ap_stack.take() { - stack - } else { - // Create a new stack - let stack = stack::alloc_stack( - KERNEL_STACK_SIZE_IN_PAGES, - &mut kernel_mmi_ref.lock().page_table, - ).ok_or("could not allocate AP stack!")?; - - ap_data.ap_stack_start.write(stack.bottom()); - ap_data.ap_stack_end.write(stack.top_unusable()); - - stack + for def_mpidr in BOARD_CONFIG.cpu_ids { + let cpu_id = CpuId::from(def_mpidr); + let mpidr = MpidrValue::from(cpu_id); + + ap_data.ap_ready.write(0); + let stack = if let Some(stack) = ap_stack.take() { + stack + } else { + // Create a new stack + let stack = stack::alloc_stack( + KERNEL_STACK_SIZE_IN_PAGES, + &mut kernel_mmi_ref.lock().page_table, + ).ok_or("could not allocate AP stack!")?; + + ap_data.ap_stack_start.write(stack.bottom()); + ap_data.ap_stack_end.write(stack.top_unusable()); + + stack + }; + + // Associate the stack to this CpuId + ap_start::insert_ap_stack(cpu_id.value(), stack); + + if let Err(kind) = cpu_on(mpidr.value(), entry_point_phys_addr.value() as _, ap_data_phys_addr.value() as _) { + let msg = match kind { + InvalidParameters => Some("InvalidParameters"), + AlreadyOn => Some("AlreadyOn"), + NotPresent => Some("NotPresent"), + Disabled => Some("Disabled"), + NotSupported => Some("NotSupported"), + Denied => Some("Denied"), + OnPending => Some("OnPending"), + InternalFailure => Some("InternalFailure"), + InvalidAddress => Some("InvalidAddress"), + _ => Some("Unknown"), }; - // Associate the stack to this CpuId - ap_start::insert_ap_stack(cpu_id.value(), stack); - - if let Err(kind) = cpu_on(mpidr.value(), entry_point_phys_addr.value() as _, ap_data_phys_addr.value() as _) { - let msg = match kind { - // normal errors (CpuId invalid or equal to the BSP one) - InvalidParameters => None, - AlreadyOn => None, - NotPresent => None, - Disabled => None, - - // unpredicted errors - NotSupported => Some("NotSupported"), - Denied => Some("Denied"), - OnPending => Some("OnPending"), - InternalFailure => Some("InternalFailure"), - InvalidAddress => Some("InvalidAddress"), - _ => Some("Unknown"), - }; - - // Re-take the stack we allocated for this CPU - // so we can reuse it the next CPU. - ap_stack = ap_start::take_ap_stack(cpu_id.value()).map(|s| s.into_inner()); - - if let Some(msg) = msg { - log::error!("Tried to start CPU core {} but got PSCI error: {}", cpu_id, msg); - } - } else { - log::info!("CpuId {:?} seems to be valid", cpu_id); - - // Wait for the core to take note of the stack boundaries - while ap_data.ap_ready.read() != 1 {} - - // ap_stack is None because the stack will be used by - // the booting core; a new one will be created for the - // next core - - // remember this CpuId - online_cores += 1; + // Re-take the stack we allocated for this CPU + // so we can reuse it the next CPU. + ap_stack = ap_start::take_ap_stack(cpu_id.value()).map(|s| s.into_inner()); + + if let Some(msg) = msg { + log::error!("Tried to start CPU core {} but got PSCI error: {}", cpu_id, msg); } + } else { + // Wait for the core to take note of the stack boundaries + while ap_data.ap_ready.read() != 1 {} + + // ap_stack is None because the stack will be used by + // the booting core; a new one will be created for the + // next core + + // remember this CpuId + online_cores += 1; } }