From dedb5e40702fee65766bbd813fc43f3935c63a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Mon, 12 Jan 2026 01:30:00 -0300 Subject: [PATCH 01/12] Added support for ELF and Mach-O AArch64 relocation specifiers for symbol access Co-authored-by: Trevor Gross --- .../src/aarch64_outline_atomics.rs | 69 +++++++++++++++++-- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index df0cf765..32455232 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -135,18 +135,73 @@ macro_rules! stxp { }; } +// The AArch64 assembly syntax for relocation specifiers +// when accessing symbols changes depending on the target executable format. +// In ELF (used in Linux), we have a prefix notation surrounded by colons (:specifier:sym), +// while in Mach-O object files (used in MacOS), a postfix notation is used (sym@specifier). + +/// AArch64 ELF position-independent addressing: +/// +/// adrp xN, symbol +/// add xN, xN, :lo12:symbol +/// +/// The :lo12: modifier selects the low 12 bits of the symbol address +/// and emits an ELF relocation such as R_AARCH64_ADD_ABS_LO12_NC. +/// +/// Defined by the AArch64 ELF psABI. +/// See: . +#[cfg(not(target_vendor = "apple"))] +macro_rules! sym { + ($sym:literal) => { + $sym + }; +} + +#[cfg(not(target_vendor = "apple"))] +macro_rules! sym_off { + ($sym:literal) => { + concat!(":lo12:", $sym) + }; +} + +/// Mach-O ARM64 relocation types: +/// ARM64_RELOC_PAGE21 +/// ARM64_RELOC_PAGEOFF12 +/// +/// These relocations implement the @PAGE / @PAGEOFF split used by +/// adrp + add sequences on Apple platforms. +/// +/// adrp xN, symbol@PAGE -> ARM64_RELOC_PAGE21 +/// add xN, xN, symbol@PAGEOFF -> ARM64_RELOC_PAGEOFF12 +/// +/// Relocation types defined by Apple in XNU: . +/// See: . +#[cfg(target_vendor = "apple")] +macro_rules! sym { + ($sym:literal) => { + concat!($sym, "@PAGE") + }; +} + +#[cfg(target_vendor = "apple")] +macro_rules! sym_off { + ($sym:literal) => { + concat!($sym, "@PAGEOFF") + }; +} + // If supported, perform the requested LSE op and return, or fallthrough. macro_rules! try_lse_op { ($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => { concat!( - ".arch_extension lse; ", - "adrp x16, {have_lse}; ", - "ldrb w16, [x16, :lo12:{have_lse}]; ", - "cbz w16, 8f; ", + ".arch_extension lse\n", + concat!("adrp x16, ", sym!("{have_lse}"), "\n"), + concat!("ldrb w16, [x16, ", sym_off!("{have_lse}"), "]\n"), + "cbz w16, 8f\n", // LSE_OP s(reg),* [$mem] - concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]; ",), - "ret; ", - "8:" + concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",), + "ret + 8:" ) }; } From f4afca9881218b034beb241709e39c7cb9d1a30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Mon, 12 Jan 2026 01:56:00 -0300 Subject: [PATCH 02/12] Remove Linux os gate --- builtins-test/tests/lse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtins-test/tests/lse.rs b/builtins-test/tests/lse.rs index 56891be8..006618f3 100644 --- a/builtins-test/tests/lse.rs +++ b/builtins-test/tests/lse.rs @@ -1,6 +1,6 @@ #![feature(decl_macro)] // so we can use pub(super) #![feature(macro_metavar_expr_concat)] -#![cfg(all(target_arch = "aarch64", target_os = "linux"))] +#![cfg(all(target_arch = "aarch64"))] /// Translate a byte size to a Rust type. macro int_ty { From 14ea37ad2104d7daa6ec84f804881a5ea2a214cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Mon, 12 Jan 2026 01:57:58 -0300 Subject: [PATCH 03/12] Fix syntax error --- builtins-test/tests/lse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtins-test/tests/lse.rs b/builtins-test/tests/lse.rs index 006618f3..854ea021 100644 --- a/builtins-test/tests/lse.rs +++ b/builtins-test/tests/lse.rs @@ -1,6 +1,6 @@ #![feature(decl_macro)] // so we can use pub(super) #![feature(macro_metavar_expr_concat)] -#![cfg(all(target_arch = "aarch64"))] +#![cfg(target_arch = "aarch64")] /// Translate a byte size to a Rust type. macro int_ty { From 8e575698f6163d662665e56656aa16ed8c463c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Mon, 12 Jan 2026 02:30:22 -0300 Subject: [PATCH 04/12] Enable aarch64 outline-atomics in tests --- compiler-builtins/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler-builtins/src/lib.rs b/compiler-builtins/src/lib.rs index c9932096..3e3f6bb5 100644 --- a/compiler-builtins/src/lib.rs +++ b/compiler-builtins/src/lib.rs @@ -55,7 +55,10 @@ pub mod arm; #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))] pub mod aarch64; -#[cfg(all(target_arch = "aarch64", target_feature = "outline-atomics"))] +#[cfg(all( + target_arch = "aarch64", + any(target_feature = "outline-atomics", feature = "mangled-names") +))] pub mod aarch64_outline_atomics; #[cfg(all( From fcffd7944b673702df35580c98970eb0299dbc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Mon, 12 Jan 2026 02:35:49 -0300 Subject: [PATCH 05/12] Added explanatory comments about the mangled-names feature gate --- compiler-builtins/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler-builtins/src/lib.rs b/compiler-builtins/src/lib.rs index 3e3f6bb5..af3ce3fb 100644 --- a/compiler-builtins/src/lib.rs +++ b/compiler-builtins/src/lib.rs @@ -55,6 +55,8 @@ pub mod arm; #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))] pub mod aarch64; +// Note that we enable the module on "mangled-names" because that is the default feature +// in the builtins-test tests. So this is a way of enabling the module during testing. #[cfg(all( target_arch = "aarch64", any(target_feature = "outline-atomics", feature = "mangled-names") From 6ccca2f5205c3390036c23b2f07d35e695b5c7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Mon, 12 Jan 2026 03:09:20 -0300 Subject: [PATCH 06/12] Improved assertions on compare-and-swap test --- builtins-test/tests/lse.rs | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/builtins-test/tests/lse.rs b/builtins-test/tests/lse.rs index 854ea021..205ed322 100644 --- a/builtins-test/tests/lse.rs +++ b/builtins-test/tests/lse.rs @@ -17,14 +17,15 @@ mod cas { fn $name() { builtins_test::fuzz_2(10000, |expected: super::int_ty!($bytes), new| { let mut target = expected.wrapping_add(10); + let ret: super::int_ty!($bytes) = unsafe { + compiler_builtins::aarch64_outline_atomics::$name::$name( + expected, + new, + &mut target, + ) + }; assert_eq!( - unsafe { - compiler_builtins::aarch64_outline_atomics::$name::$name( - expected, - new, - &mut target, - ) - }, + ret, expected.wrapping_add(10), "return value should always be the previous value", ); @@ -35,15 +36,17 @@ mod cas { ); target = expected; + let ret: super::int_ty!($bytes) = unsafe { + compiler_builtins::aarch64_outline_atomics::$name::$name( + expected, + new, + &mut target, + ) + }; assert_eq!( - unsafe { - compiler_builtins::aarch64_outline_atomics::$name::$name( - expected, - new, - &mut target, - ) - }, - expected + ret, + expected, + "the new return value should always be the previous value (i.e. the first parameter passed to the function)", ); assert_eq!(target, new, "should have updated target"); }); From efeb4dfe689dc1c269d2fc29865edac5f14681dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Thu, 22 Jan 2026 00:39:03 -0300 Subject: [PATCH 07/12] fix: added sign extension to emitted asm --- .../src/aarch64_outline_atomics.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index 32455232..e7e3f1f3 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -190,6 +190,24 @@ macro_rules! sym_off { }; } +/// Emit the required AArch64 sign-extension for signed integer values +/// smaller than 32 bits, operating in-place on the given register. +/// +/// Intended for use in naked assembly before `ret` or before a value +/// is compared/consumed at full register width. +#[rustfmt::skip] +macro_rules! sign_extend { + (1) => { + concat!("sxtb ", reg!(1, 0), ", ", reg!(1, 0)) + }; + (2) => { + concat!("sxth ", reg!(2, 0), ", ", reg!(2, 0)) + }; + (4) => { "" }; + (8) => { "" }; + (16) => { "" }; +} + // If supported, perform the requested LSE op and return, or fallthrough. macro_rules! try_lse_op { ($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => { @@ -250,6 +268,8 @@ macro_rules! compare_and_swap { concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"), "cbnz w17, 0b", "1:", + // SXTB s(0), s(0) + sign_extend!($bytes), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -310,6 +330,8 @@ macro_rules! swap { // STXR w(tmp1), s(tmp0), [x1] concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), "cbnz w17, 0b", + // SXTB s(0), s(0) + sign_extend!($bytes), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -340,6 +362,8 @@ macro_rules! fetch_op { // STXR w(tmp2), s(tmp1), [x1] concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), "cbnz w15, 0b", + // SXTB s(0), s(0) + sign_extend!($bytes), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } From 45f21ff88775892ab3d83a986650a07fe10f2708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Thu, 22 Jan 2026 01:04:59 -0300 Subject: [PATCH 08/12] fix: sign-extend for LSE path as well --- compiler-builtins/src/aarch64_outline_atomics.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index e7e3f1f3..db0f1ea6 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -218,8 +218,10 @@ macro_rules! try_lse_op { "cbz w16, 8f\n", // LSE_OP s(reg),* [$mem] concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",), - "ret - 8:" + "ret\n", + // SXTB s(0), s(0) + concat!(sign_extend!($bytes), "\n"), + "8:" ) }; } From 1060b18466dda3ebca1eae3e17e644bc03fe5639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Thu, 22 Jan 2026 20:37:43 -0300 Subject: [PATCH 09/12] fix incorrect order of sign extension --- compiler-builtins/src/aarch64_outline_atomics.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index db0f1ea6..779da940 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -218,10 +218,10 @@ macro_rules! try_lse_op { "cbz w16, 8f\n", // LSE_OP s(reg),* [$mem] concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",), - "ret\n", // SXTB s(0), s(0) concat!(sign_extend!($bytes), "\n"), - "8:" + "ret + 8:" ) }; } From 8285fe118bf05e4543fa3e66cf2eb7f413f87713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Thu, 22 Jan 2026 20:43:05 -0300 Subject: [PATCH 10/12] improve sign_extend macro --- .../src/aarch64_outline_atomics.rs | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index 779da940..c7258ec4 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -190,22 +190,25 @@ macro_rules! sym_off { }; } -/// Emit the required AArch64 sign-extension for signed integer values +/// Given a byte size and a register number given by $num, +/// this will emit the required AArch64 sign-extension for signed integer values /// smaller than 32 bits, operating in-place on the given register. /// +/// If the byte size is bigger than 2, this turns into a no-op. +/// /// Intended for use in naked assembly before `ret` or before a value /// is compared/consumed at full register width. #[rustfmt::skip] macro_rules! sign_extend { - (1) => { - concat!("sxtb ", reg!(1, 0), ", ", reg!(1, 0)) + (1, $num:literal) => { + concat!("sxtb ", reg!(1, $num), ", ", reg!(1, $num)) }; - (2) => { - concat!("sxth ", reg!(2, 0), ", ", reg!(2, 0)) + (2, $num:literal) => { + concat!("sxth ", reg!(2, $num), ", ", reg!(2, $num)) }; - (4) => { "" }; - (8) => { "" }; - (16) => { "" }; + (4, $num:literal) => { "" }; + (8, $num:literal) => { "" }; + (16, $num:literal) => { "" }; } // If supported, perform the requested LSE op and return, or fallthrough. @@ -219,7 +222,7 @@ macro_rules! try_lse_op { // LSE_OP s(reg),* [$mem] concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",), // SXTB s(0), s(0) - concat!(sign_extend!($bytes), "\n"), + concat!(sign_extend!($bytes, 0), "\n"), "ret 8:" ) @@ -271,7 +274,7 @@ macro_rules! compare_and_swap { "cbnz w17, 0b", "1:", // SXTB s(0), s(0) - sign_extend!($bytes), + sign_extend!($bytes, 0), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -333,7 +336,7 @@ macro_rules! swap { concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), "cbnz w17, 0b", // SXTB s(0), s(0) - sign_extend!($bytes), + sign_extend!($bytes, 0), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -365,7 +368,7 @@ macro_rules! fetch_op { concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), "cbnz w15, 0b", // SXTB s(0), s(0) - sign_extend!($bytes), + sign_extend!($bytes, 0), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } From 66b43e2d92c27072019e8d7b319acf1cd3754780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Fri, 23 Jan 2026 19:10:39 -0300 Subject: [PATCH 11/12] revert sign extension changes & change function declarations to unsigned integer types --- builtins-test/tests/lse.rs | 10 ++-- .../src/aarch64_outline_atomics.rs | 49 ++++--------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/builtins-test/tests/lse.rs b/builtins-test/tests/lse.rs index 205ed322..e4dedb0e 100644 --- a/builtins-test/tests/lse.rs +++ b/builtins-test/tests/lse.rs @@ -4,11 +4,11 @@ /// Translate a byte size to a Rust type. macro int_ty { - (1) => { i8 }, - (2) => { i16 }, - (4) => { i32 }, - (8) => { i64 }, - (16) => { i128 } + (1) => { u8 }, + (2) => { u16 }, + (4) => { u32 }, + (8) => { u64 }, + (16) => { u128 } } mod cas { diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index c7258ec4..54232fe5 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -37,11 +37,11 @@ intrinsics! { /// Translate a byte size to a Rust type. #[rustfmt::skip] macro_rules! int_ty { - (1) => { i8 }; - (2) => { i16 }; - (4) => { i32 }; - (8) => { i64 }; - (16) => { i128 }; + (1) => { u8 }; + (2) => { u16 }; + (4) => { u32 }; + (8) => { u64 }; + (16) => { u128 }; } /// Given a byte size and a register number, return a register of the appropriate size. @@ -190,27 +190,6 @@ macro_rules! sym_off { }; } -/// Given a byte size and a register number given by $num, -/// this will emit the required AArch64 sign-extension for signed integer values -/// smaller than 32 bits, operating in-place on the given register. -/// -/// If the byte size is bigger than 2, this turns into a no-op. -/// -/// Intended for use in naked assembly before `ret` or before a value -/// is compared/consumed at full register width. -#[rustfmt::skip] -macro_rules! sign_extend { - (1, $num:literal) => { - concat!("sxtb ", reg!(1, $num), ", ", reg!(1, $num)) - }; - (2, $num:literal) => { - concat!("sxth ", reg!(2, $num), ", ", reg!(2, $num)) - }; - (4, $num:literal) => { "" }; - (8, $num:literal) => { "" }; - (16, $num:literal) => { "" }; -} - // If supported, perform the requested LSE op and return, or fallthrough. macro_rules! try_lse_op { ($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => { @@ -221,8 +200,6 @@ macro_rules! try_lse_op { "cbz w16, 8f\n", // LSE_OP s(reg),* [$mem] concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",), - // SXTB s(0), s(0) - concat!(sign_extend!($bytes, 0), "\n"), "ret 8:" ) @@ -273,8 +250,6 @@ macro_rules! compare_and_swap { concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"), "cbnz w17, 0b", "1:", - // SXTB s(0), s(0) - sign_extend!($bytes, 0), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -283,15 +258,15 @@ macro_rules! compare_and_swap { }; } -// i128 uses a completely different impl, so it has its own macro. -macro_rules! compare_and_swap_i128 { +// u128 uses a completely different impl, so it has its own macro. +macro_rules! compare_and_swap_u128 { ($ordering:ident, $name:ident) => { intrinsics! { #[maybe_use_optimized_c_shim] #[unsafe(naked)] pub unsafe extern "C" fn $name ( - expected: i128, desired: i128, ptr: *mut i128 - ) -> i128 { + expected: u128, desired: u128, ptr: *mut u128 + ) -> u128 { core::arch::naked_asm! { // CASP x0, x1, x2, x3, [x4]; if LSE supported. try_lse_op!("cas", $ordering, 16, 0, 1, 2, 3, [x4]), @@ -335,8 +310,6 @@ macro_rules! swap { // STXR w(tmp1), s(tmp0), [x1] concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), "cbnz w17, 0b", - // SXTB s(0), s(0) - sign_extend!($bytes, 0), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -367,8 +340,6 @@ macro_rules! fetch_op { // STXR w(tmp2), s(tmp1), [x1] concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), "cbnz w15, 0b", - // SXTB s(0), s(0) - sign_extend!($bytes, 0), "ret", have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } @@ -475,7 +446,7 @@ macro_rules! foreach_ldset { } foreach_cas!(compare_and_swap); -foreach_cas16!(compare_and_swap_i128); +foreach_cas16!(compare_and_swap_u128); foreach_swp!(swap); foreach_ldadd!(add); foreach_ldclr!(and); From b1019f58fc5527dc0028499fdc312c79adab3634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Freitas?= Date: Fri, 23 Jan 2026 20:12:19 -0300 Subject: [PATCH 12/12] Updated tests to be more comprehensive, enabling/disabling LSE. Co-authored-by: Trevor Gross --- builtins-test/tests/lse.rs | 69 +++++++++++++++---- .../src/aarch64_outline_atomics.rs | 13 ++++ 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/builtins-test/tests/lse.rs b/builtins-test/tests/lse.rs index e4dedb0e..03fe9467 100644 --- a/builtins-test/tests/lse.rs +++ b/builtins-test/tests/lse.rs @@ -2,6 +2,45 @@ #![feature(macro_metavar_expr_concat)] #![cfg(target_arch = "aarch64")] +use std::sync::Mutex; + +use compiler_builtins::aarch64_outline_atomics::{get_have_lse_atomics, set_have_lse_atomics}; +use compiler_builtins::int::{Int, MinInt}; +use compiler_builtins::{foreach_bytes, foreach_ordering}; + +#[track_caller] +fn with_maybe_lse_atomics(use_lse: bool, f: impl FnOnce()) { + // Ensure tests run in parallel don't interleave global settings + static LOCK: Mutex<()> = Mutex::new(()); + let _g = LOCK.lock().unwrap(); + let old = get_have_lse_atomics(); + // safety: as the caller of the unsafe fn `set_have_lse_atomics`, we + // have to ensure the CPU supports LSE. This is why we make this assertion. + if use_lse || old { + assert!(std::arch::is_aarch64_feature_detected!("lse")); + } + unsafe { set_have_lse_atomics(use_lse) }; + f(); + unsafe { set_have_lse_atomics(old) }; +} + +pub fn run_fuzz_tests_with_lse_variants(n: u32, f: F) +where + ::Unsigned: Int, +{ + // We use `fuzz_2` because our subject function `f` requires two inputs + let test_fn = || { + builtins_test::fuzz_2(n, f); + }; + // Always run without LSE + with_maybe_lse_atomics(false, test_fn); + + // Conditionally run with LSE + if std::arch::is_aarch64_feature_detected!("lse") { + with_maybe_lse_atomics(true, test_fn); + } +} + /// Translate a byte size to a Rust type. macro int_ty { (1) => { u8 }, @@ -15,7 +54,7 @@ mod cas { pub(super) macro test($_ordering:ident, $bytes:tt, $name:ident) { #[test] fn $name() { - builtins_test::fuzz_2(10000, |expected: super::int_ty!($bytes), new| { + crate::run_fuzz_tests_with_lse_variants(10000, |expected: super::int_ty!($bytes), new| { let mut target = expected.wrapping_add(10); let ret: super::int_ty!($bytes) = unsafe { compiler_builtins::aarch64_outline_atomics::$name::$name( @@ -62,16 +101,21 @@ mod swap { pub(super) macro test($_ordering:ident, $bytes:tt, $name:ident) { #[test] fn $name() { - builtins_test::fuzz_2(10000, |left: super::int_ty!($bytes), mut right| { - let orig_right = right; - assert_eq!( - unsafe { - compiler_builtins::aarch64_outline_atomics::$name::$name(left, &mut right) - }, - orig_right - ); - assert_eq!(left, right); - }); + crate::run_fuzz_tests_with_lse_variants( + 10000, + |left: super::int_ty!($bytes), mut right| { + let orig_right = right; + assert_eq!( + unsafe { + compiler_builtins::aarch64_outline_atomics::$name::$name( + left, &mut right, + ) + }, + orig_right + ); + assert_eq!(left, right); + }, + ); } } } @@ -83,7 +127,7 @@ macro_rules! test_op { ($_ordering:ident, $bytes:tt, $name:ident) => { #[test] fn $name() { - builtins_test::fuzz_2(10000, |old, val| { + crate::run_fuzz_tests_with_lse_variants(10000, |old, val| { let mut target = old; let op: fn(super::int_ty!($bytes), super::int_ty!($bytes)) -> _ = $($op)*; let expected = op(old, val); @@ -101,7 +145,6 @@ test_op!(add, |left, right| left.wrapping_add(right)); test_op!(clr, |left, right| left & !right); test_op!(xor, std::ops::BitXor::bitxor); test_op!(or, std::ops::BitOr::bitor); -use compiler_builtins::{foreach_bytes, foreach_ordering}; compiler_builtins::foreach_cas!(cas::test); compiler_builtins::foreach_cas16!(test_cas16); compiler_builtins::foreach_swp!(swap::test); diff --git a/compiler-builtins/src/aarch64_outline_atomics.rs b/compiler-builtins/src/aarch64_outline_atomics.rs index 54232fe5..100b6715 100644 --- a/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/compiler-builtins/src/aarch64_outline_atomics.rs @@ -34,6 +34,19 @@ intrinsics! { } } +/// Function to enable/disable LSE. To be used only for testing purposes. +#[cfg(feature = "mangled-names")] +pub unsafe fn set_have_lse_atomics(has_lse: bool) { + let lse_flag = if has_lse { 1 } else { 0 }; + HAVE_LSE_ATOMICS.store(lse_flag, Ordering::Relaxed); +} + +/// Function to obtain whether LSE is enabled or not. To be used only for testing purposes. +#[cfg(feature = "mangled-names")] +pub fn get_have_lse_atomics() -> bool { + HAVE_LSE_ATOMICS.load(Ordering::Relaxed) != 0 +} + /// Translate a byte size to a Rust type. #[rustfmt::skip] macro_rules! int_ty {