diff --git a/src/aarch64/config.rs b/src/aarch64/config.rs index 7a0ddd1..fb16b48 100644 --- a/src/aarch64/config.rs +++ b/src/aarch64/config.rs @@ -1,4 +1,4 @@ -pub const VVAR_PAGES: usize = 5; +pub const VVAR_PAGES: usize = 4; #[repr(i32)] pub enum ClockMode { diff --git a/src/aarch64/vdso_data.rs b/src/aarch64/vdso_data.rs index 566a51a..cad2b17 100644 --- a/src/aarch64/vdso_data.rs +++ b/src/aarch64/vdso_data.rs @@ -1,20 +1,10 @@ -use axplat::time::{ - NANOS_PER_SEC, current_ticks, monotonic_time_nanos, nanos_to_ticks, wall_time_nanos, -}; - -use super::config::ClockMode; -use crate::update::{VdsoClock, clocks_calc_mult_shift, update_vdso_clock}; - +use crate::vdso_time_data::VdsoTimeData; #[repr(C)] -#[repr(align(4096))] pub struct VdsoData { - pub clock_page0: VdsoClock, - pub _unused: [u8; 1648], - pub tz_minuteswest: i32, - pub tz_dsttime: i32, - pub hrtimer_res: u32, - pub _pad: [u8; 4096 - 1956], - pub clock_page1: VdsoClock, + pub time_data: VdsoTimeData, + pub timen_data: [u8; 4096], + pub rng_data: [u8; 4096], + pub arch_data: [u8; 4096], } impl Default for VdsoData { @@ -26,46 +16,14 @@ impl Default for VdsoData { impl VdsoData { pub const fn new() -> Self { Self { - clock_page0: VdsoClock::new(), - _unused: [0; 1648], - tz_minuteswest: 0, - tz_dsttime: 0, - hrtimer_res: 1, - _pad: [0; 4096 - 1956], - clock_page1: VdsoClock::new(), + time_data: VdsoTimeData::new(), + timen_data: [0u8; 4096], + rng_data: [0u8; 4096], + arch_data: [0u8; 4096], } } - pub fn update(&mut self) { - let wall_ns = wall_time_nanos(); - let mono_ns = monotonic_time_nanos(); - - let ticks_per_sec = nanos_to_ticks(NANOS_PER_SEC); - let mult_shift = clocks_calc_mult_shift(ticks_per_sec, NANOS_PER_SEC, 10); - let cycle_now = current_ticks(); - - self.clock_page0.write_seqcount_begin(); - self.clock_page0.clock_mode = ClockMode::Cntvct as i32; - self.clock_page0.mask = u64::MAX; - update_vdso_clock( - &mut self.clock_page0, - cycle_now, - wall_ns, - mono_ns, - mult_shift, - ); - self.clock_page0.write_seqcount_end(); - - self.clock_page1.write_seqcount_begin(); - self.clock_page1.clock_mode = ClockMode::Cntvct as i32; - self.clock_page1.mask = u64::MAX; - update_vdso_clock( - &mut self.clock_page1, - cycle_now, - wall_ns, - mono_ns, - mult_shift, - ); - self.clock_page1.write_seqcount_end(); + pub fn time_update(&mut self) { + self.time_data.update(); } } diff --git a/src/lib.rs b/src/lib.rs index 4e7f311..2332cc7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ pub mod embed; pub mod guard; mod update; pub mod vdso; +mod vdso_time_data; cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { diff --git a/src/loongarch64/config.rs b/src/loongarch64/config.rs index 3396cda..ffd7a09 100644 --- a/src/loongarch64/config.rs +++ b/src/loongarch64/config.rs @@ -1,5 +1,5 @@ -pub const VVAR_PAGES: usize = 44; -#[cfg(target_arch = "loongarch64")] +pub const VVAR_PAGES: usize = 20; + #[repr(i32)] pub enum ClockMode { None, diff --git a/src/loongarch64/vdso_data.rs b/src/loongarch64/vdso_data.rs index 167a1b5..0fc72c8 100644 --- a/src/loongarch64/vdso_data.rs +++ b/src/loongarch64/vdso_data.rs @@ -1,43 +1,24 @@ -use axplat::time::{ - NANOS_PER_SEC, current_ticks, monotonic_time_nanos, nanos_to_ticks, wall_time_nanos, -}; - -use crate::update::{VdsoClock, clocks_calc_mult_shift, update_vdso_clock}; +use crate::vdso_time_data::VdsoTimeData; #[repr(C)] -#[repr(align(4096))] -#[derive(Default)] pub struct VdsoData { - pub clocks: [VdsoClock; 2], - pub tz_minuteswest: i32, - pub tz_dsttime: i32, - pub hrtimer_res: u32, + pub time_data: VdsoTimeData, +} + +impl Default for VdsoData { + fn default() -> Self { + Self::new() + } } impl VdsoData { - /// Create a new VdsoData with default values. pub const fn new() -> Self { Self { - clocks: [VdsoClock::new(), VdsoClock::new()], - tz_minuteswest: 0, - tz_dsttime: 0, - hrtimer_res: 1, + time_data: VdsoTimeData::new(), } } - /// Update vDSO clocks and basetimes. - pub fn update(&mut self) { - let cycle_now = current_ticks(); - let wall_ns = wall_time_nanos(); - let mono_ns = monotonic_time_nanos(); - - let ticks_per_sec = nanos_to_ticks(NANOS_PER_SEC); - let mult_shift = clocks_calc_mult_shift(ticks_per_sec, NANOS_PER_SEC, 10); - - for clk in &mut self.clocks { - clk.write_seqcount_begin(); - update_vdso_clock(clk, cycle_now, wall_ns, mono_ns, mult_shift); - clk.write_seqcount_end(); - } + pub fn time_update(&mut self) { + self.time_data.update(); } } diff --git a/src/riscv64/config.rs b/src/riscv64/config.rs index 4c9bed9..fa7fcfc 100644 --- a/src/riscv64/config.rs +++ b/src/riscv64/config.rs @@ -1,4 +1,4 @@ -pub const VVAR_PAGES: usize = 2; +pub const VVAR_PAGES: usize = 4; #[repr(i32)] pub enum ClockMode { diff --git a/src/riscv64/vdso_data.rs b/src/riscv64/vdso_data.rs index 5dce2db..0fc72c8 100644 --- a/src/riscv64/vdso_data.rs +++ b/src/riscv64/vdso_data.rs @@ -1,41 +1,24 @@ -use axplat::time::{ - NANOS_PER_SEC, current_ticks, monotonic_time_nanos, nanos_to_ticks, wall_time_nanos, -}; - -use crate::update::{VdsoClock, clocks_calc_mult_shift, update_vdso_clock}; +use crate::vdso_time_data::VdsoTimeData; #[repr(C)] -#[repr(align(4096))] -#[derive(Default)] pub struct VdsoData { - pub clocks: [VdsoClock; 2], - pub tz_minuteswest: i32, - pub tz_dsttime: i32, - pub hrtimer_res: u32, + pub time_data: VdsoTimeData, +} + +impl Default for VdsoData { + fn default() -> Self { + Self::new() + } } impl VdsoData { pub const fn new() -> Self { Self { - clocks: [VdsoClock::new(), VdsoClock::new()], - tz_minuteswest: 0, - tz_dsttime: 0, - hrtimer_res: 1, + time_data: VdsoTimeData::new(), } } - pub fn update(&mut self) { - let cycle_now = current_ticks(); - let wall_ns = wall_time_nanos(); - let mono_ns = monotonic_time_nanos(); - - let ticks_per_sec = nanos_to_ticks(NANOS_PER_SEC); - let mult_shift = clocks_calc_mult_shift(ticks_per_sec, NANOS_PER_SEC, 10); - - for clk in &mut self.clocks { - clk.write_seqcount_begin(); - update_vdso_clock(clk, cycle_now, wall_ns, mono_ns, mult_shift); - clk.write_seqcount_end(); - } + pub fn time_update(&mut self) { + self.time_data.update(); } } diff --git a/src/update.rs b/src/update.rs index 062ad89..02cf898 100644 --- a/src/update.rs +++ b/src/update.rs @@ -1,11 +1,8 @@ -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::Ordering; use axplat::time::NANOS_PER_SEC; -use crate::config::ClockMode; - -/// Number of clock bases -const VDSO_BASES: usize = 16; +use crate::{config::ClockMode, vdso_time_data::VdsoClock}; /// vDSO timestamp structure #[repr(C)] @@ -24,47 +21,6 @@ impl VdsoTimestamp { } } -#[repr(C)] -#[derive(Default)] -pub struct VdsoClock { - pub seq: AtomicU32, - pub clock_mode: i32, - pub cycle_last: AtomicU64, - pub mask: u64, - pub mult: u32, - pub shift: u32, - pub basetime: [VdsoTimestamp; VDSO_BASES], - pub _unused: u32, -} - -impl VdsoClock { - /// Create a new VdsoClock with default values. - pub const fn new() -> Self { - Self { - seq: AtomicU32::new(0), - clock_mode: 1, - cycle_last: AtomicU64::new(0), - mask: u64::MAX, - mult: 0, - shift: 32, - basetime: [VdsoTimestamp::new(); VDSO_BASES], - _unused: 0, - } - } - - pub fn write_seqcount_begin(&self) { - let seq = self.seq.load(Ordering::Relaxed); - self.seq.store(seq.wrapping_add(1), Ordering::Release); - core::sync::atomic::fence(Ordering::SeqCst); - } - - pub fn write_seqcount_end(&self) { - core::sync::atomic::fence(Ordering::SeqCst); - let seq = self.seq.load(Ordering::Relaxed); - self.seq.store(seq.wrapping_add(1), Ordering::Release); - } -} - /// Update vDSO clock. pub fn update_vdso_clock( clk: &mut VdsoClock, @@ -74,10 +30,10 @@ pub fn update_vdso_clock( mult_shift: (u32, u32), ) { let prev_cycle = clk.cycle_last.load(Ordering::Relaxed); - let prev_basetime_ns = clk.basetime[1] + let prev_basetime_ns = clk.time_data[1] .sec .wrapping_mul(NANOS_PER_SEC) - .wrapping_add(clk.basetime[1].nsec); + .wrapping_add(clk.time_data[1].nsec); // Check if this is a counter-based clock mode (non-None) let is_counter_mode = clk.clock_mode != (ClockMode::None as i32); @@ -89,16 +45,16 @@ pub fn update_vdso_clock( let (mult, shift) = mult_shift; clk.mult = mult; clk.shift = shift; - clk.basetime[1].sec = mono_ns / NANOS_PER_SEC; - clk.basetime[1].nsec = (mono_ns % NANOS_PER_SEC) << shift; + clk.time_data[1].sec = mono_ns / NANOS_PER_SEC; + clk.time_data[1].nsec = (mono_ns % NANOS_PER_SEC) << shift; clk.cycle_last.store(cycle_now, Ordering::Relaxed); } else { let (mult, shift) = mult_shift; if !(mult == u32::MAX && shift == 0) { clk.mult = mult; clk.shift = shift; - clk.basetime[1].sec = mono_ns / NANOS_PER_SEC; - clk.basetime[1].nsec = (mono_ns % NANOS_PER_SEC) << shift; + clk.time_data[1].sec = mono_ns / NANOS_PER_SEC; + clk.time_data[1].nsec = (mono_ns % NANOS_PER_SEC) << shift; clk.cycle_last.store(cycle_now, Ordering::Relaxed); } else { let delta_cycles = (cycle_now.wrapping_sub(prev_cycle)) & clk.mask; @@ -107,8 +63,8 @@ pub fn update_vdso_clock( let (mult, shift) = clocks_calc_mult_shift(delta_cycles, delta_ns, 1); clk.mult = mult; clk.shift = shift; - clk.basetime[1].sec = mono_ns / NANOS_PER_SEC; - clk.basetime[1].nsec = (mono_ns % NANOS_PER_SEC) << shift; + clk.time_data[1].sec = mono_ns / NANOS_PER_SEC; + clk.time_data[1].nsec = (mono_ns % NANOS_PER_SEC) << shift; clk.cycle_last.store(cycle_now, Ordering::Relaxed); } } @@ -116,17 +72,17 @@ pub fn update_vdso_clock( } else { // ClockMode::None - No cycle->ns conversion; store direct monotonic ns. clk.mult = 0; - clk.basetime[1].sec = mono_ns / NANOS_PER_SEC; - clk.basetime[1].nsec = mono_ns % NANOS_PER_SEC; + clk.time_data[1].sec = mono_ns / NANOS_PER_SEC; + clk.time_data[1].nsec = mono_ns % NANOS_PER_SEC; clk.cycle_last.store(0, Ordering::Relaxed); } // Update realtime and boottime entries. let shift = clk.shift; - clk.basetime[0].sec = wall_ns / NANOS_PER_SEC; - clk.basetime[0].nsec = (wall_ns % NANOS_PER_SEC) << shift; - clk.basetime[7].sec = clk.basetime[1].sec; - clk.basetime[7].nsec = clk.basetime[1].nsec; + clk.time_data[0].sec = wall_ns / NANOS_PER_SEC; + clk.time_data[0].nsec = (wall_ns % NANOS_PER_SEC) << shift; + clk.time_data[7].sec = clk.time_data[1].sec; + clk.time_data[7].nsec = clk.time_data[1].nsec; if clk.seq.load(Ordering::Relaxed) < 10 { let cycle_val = clk.cycle_last.load(Ordering::Relaxed); diff --git a/src/vdso.rs b/src/vdso.rs index 3b1081c..dae4195 100644 --- a/src/vdso.rs +++ b/src/vdso.rs @@ -17,8 +17,13 @@ pub static mut VDSO_DATA: crate::vdso_data::VdsoData = crate::vdso_data::VdsoDat pub fn init_vdso_data() { unsafe { let data_ptr = core::ptr::addr_of_mut!(VDSO_DATA); - (*data_ptr).update(); + (*data_ptr).time_update(); log::info!("vDSO data initialized at {:#x}", data_ptr as usize); + #[cfg(target_arch = "aarch64")] + { + enable_cntvct_access(); + log::info!("vDSO CNTVCT access enabled"); + } } } @@ -26,7 +31,7 @@ pub fn init_vdso_data() { pub fn update_vdso_data() { unsafe { let data_ptr = core::ptr::addr_of_mut!(VDSO_DATA); - (*data_ptr).update(); + (*data_ptr).time_update(); } } diff --git a/src/vdso_time_data.rs b/src/vdso_time_data.rs new file mode 100644 index 0000000..7a3077a --- /dev/null +++ b/src/vdso_time_data.rs @@ -0,0 +1,97 @@ +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +use axplat::time::{ + NANOS_PER_SEC, current_ticks, monotonic_time_nanos, nanos_to_ticks, wall_time_nanos, +}; + +use crate::update::{VdsoTimestamp, clocks_calc_mult_shift, update_vdso_clock}; + +const VDSO_BASES: usize = 12; + +#[repr(C)] +pub struct VdsoClock { + pub seq: AtomicU32, + pub clock_mode: i32, + pub cycle_last: AtomicU64, + #[cfg(target_arch = "x86_64")] + pub max_cycles: u64, + pub mask: u64, + pub mult: u32, + pub shift: u32, + pub time_data: [VdsoTimestamp; VDSO_BASES], + pub _unused: u32, +} + +impl VdsoClock { + /// Create a new VdsoClock with default values. + pub const fn new() -> Self { + Self { + seq: AtomicU32::new(0), + clock_mode: 1, + cycle_last: AtomicU64::new(0), + // only for x86 because CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT + #[cfg(target_arch = "x86_64")] + max_cycles: 0, + + mask: u64::MAX, + mult: 0, + shift: 32, + time_data: [VdsoTimestamp::new(); VDSO_BASES], + _unused: 0, + } + } + + pub fn write_seqcount_begin(&self) { + let seq = self.seq.load(Ordering::Relaxed); + self.seq.store(seq.wrapping_add(1), Ordering::Release); + core::sync::atomic::fence(Ordering::SeqCst); + } + + pub fn write_seqcount_end(&self) { + core::sync::atomic::fence(Ordering::SeqCst); + let seq = self.seq.load(Ordering::Relaxed); + self.seq.store(seq.wrapping_add(1), Ordering::Release); + } +} + +#[repr(C)] +#[repr(align(4096))] +pub struct VdsoTimeData { + pub clock_data: [VdsoClock; 2], + pub tz_minuteswest: i32, + pub tz_dsttime: i32, + pub hrtimer_res: u32, + pub __unused: u32, +} + +impl Default for VdsoTimeData { + fn default() -> Self { + Self::new() + } +} + +impl VdsoTimeData { + pub const fn new() -> Self { + Self { + clock_data: [VdsoClock::new(), VdsoClock::new()], + tz_minuteswest: 0, + tz_dsttime: 0, + hrtimer_res: 1, + __unused: 0, + } + } + + pub fn update(&mut self) { + let cycle_now = current_ticks(); + let wall_ns = wall_time_nanos(); + let mono_ns = monotonic_time_nanos(); + let ticks_per_sec = nanos_to_ticks(NANOS_PER_SEC); + let mult_shift = clocks_calc_mult_shift(ticks_per_sec, NANOS_PER_SEC, 10); + + for clk in self.clock_data.iter_mut() { + clk.write_seqcount_begin(); + update_vdso_clock(clk, cycle_now, wall_ns, mono_ns, mult_shift); + clk.write_seqcount_end(); + } + } +} diff --git a/src/x86_64/config.rs b/src/x86_64/config.rs index ff4573d..53bfc04 100644 --- a/src/x86_64/config.rs +++ b/src/x86_64/config.rs @@ -1,4 +1,4 @@ -pub const VVAR_PAGES: usize = 4; +pub const VVAR_PAGES: usize = 6; #[repr(i32)] pub enum ClockMode { diff --git a/src/x86_64/vdso_data.rs b/src/x86_64/vdso_data.rs index db53c77..0fc72c8 100644 --- a/src/x86_64/vdso_data.rs +++ b/src/x86_64/vdso_data.rs @@ -1,21 +1,10 @@ -use axplat::time::{ - NANOS_PER_SEC, current_ticks, monotonic_time_nanos, nanos_to_ticks, wall_time_nanos, -}; - -use super::config::ClockMode; -use crate::update::{VdsoClock, clocks_calc_mult_shift, update_vdso_clock}; +use crate::vdso_time_data::VdsoTimeData; #[repr(C)] -#[repr(align(4096))] pub struct VdsoData { - pub _pad: [u8; 128], - pub clocks: [VdsoClock; 2], - pub tz_minuteswest: i32, - pub tz_dsttime: i32, - pub hrtimer_res: u32, + pub time_data: VdsoTimeData, } -#[cfg(target_arch = "x86_64")] impl Default for VdsoData { fn default() -> Self { Self::new() @@ -25,31 +14,11 @@ impl Default for VdsoData { impl VdsoData { pub const fn new() -> Self { Self { - _pad: [0; 128], - clocks: [VdsoClock::new(), VdsoClock::new()], - tz_minuteswest: 0, - tz_dsttime: 0, - hrtimer_res: 1, + time_data: VdsoTimeData::new(), } } - pub fn update(&mut self) { - let cycle_now = current_ticks(); - - let wall_ns = wall_time_nanos(); - let mono_ns = monotonic_time_nanos(); - - let ticks_per_sec = nanos_to_ticks(NANOS_PER_SEC); - let mult_shift = clocks_calc_mult_shift(ticks_per_sec, NANOS_PER_SEC, 10); - - for clk in self.clocks.iter_mut() { - clk.write_seqcount_begin(); - - clk.clock_mode = self::ClockMode::Tsc as i32; - clk.mask = u64::MAX; - update_vdso_clock(clk, cycle_now, wall_ns, mono_ns, mult_shift); - - clk.write_seqcount_end(); - } + pub fn time_update(&mut self) { + self.time_data.update(); } } diff --git a/vdso/vdso_aarch64.so b/vdso/vdso_aarch64.so old mode 100644 new mode 100755 index ebabc81..e7866b9 Binary files a/vdso/vdso_aarch64.so and b/vdso/vdso_aarch64.so differ diff --git a/vdso/vdso_loongarch64.so b/vdso/vdso_loongarch64.so old mode 100644 new mode 100755 index d42174a..b0d38aa Binary files a/vdso/vdso_loongarch64.so and b/vdso/vdso_loongarch64.so differ diff --git a/vdso/vdso_riscv64.so b/vdso/vdso_riscv64.so old mode 100644 new mode 100755 index 17fc95e..c6eccad Binary files a/vdso/vdso_riscv64.so and b/vdso/vdso_riscv64.so differ diff --git a/vdso/vdso_x86_64.so b/vdso/vdso_x86_64.so old mode 100644 new mode 100755 index f805260..094aae8 Binary files a/vdso/vdso_x86_64.so and b/vdso/vdso_x86_64.so differ