Fix generated assembly for the outline-atomics feature in Apple AArch64 targets

builtins-test/tests/lse.rs

@@ -1,6 +1,6 @@
 #![feature(decl_macro)] // so we can use pub(super)
 #![feature(macro_metavar_expr_concat)]
-#![cfg(all(target_arch = "aarch64", target_os = "linux"))]
+#![cfg(target_arch = "aarch64")]
 
 /// Translate a byte size to a Rust type.
 macro int_ty {
@@ -17,14 +17,15 @@ mod cas {
         fn $name() {
             builtins_test::fuzz_2(10000, |expected: super::int_ty!($bytes), new| {
                 let mut target = expected.wrapping_add(10);
+                let ret: super::int_ty!($bytes) = unsafe {
+                    compiler_builtins::aarch64_outline_atomics::$name::$name(
+                        expected,
+                        new,
+                        &mut target,
+                    )
+                };
                 assert_eq!(
-                    unsafe {
-                        compiler_builtins::aarch64_outline_atomics::$name::$name(
-                            expected,
-                            new,
-                            &mut target,
-                        )
-                    },
+                    ret,
                     expected.wrapping_add(10),
                     "return value should always be the previous value",
                 );
@@ -35,15 +36,17 @@ mod cas {
                 );
 
                 target = expected;
+                let ret: super::int_ty!($bytes) = unsafe {
+                    compiler_builtins::aarch64_outline_atomics::$name::$name(
+                        expected,
+                        new,
+                        &mut target,
+                    )
+                };
                 assert_eq!(
-                    unsafe {
-                        compiler_builtins::aarch64_outline_atomics::$name::$name(
-                            expected,
-                            new,
-                            &mut target,
-                        )
-                    },
-                    expected
+                    ret,
+                    expected,
+                    "the new return value should always be the previous value (i.e. the first parameter passed to the function)",
                 );
                 assert_eq!(target, new, "should have updated target");
             });

compiler-builtins/src/aarch64_outline_atomics.rs

@@ -135,17 +135,92 @@ macro_rules! stxp {
     };
 }
 
+// The AArch64 assembly syntax for relocation specifiers
+// when accessing symbols changes depending on the target executable format.
+// In ELF (used in Linux), we have a prefix notation surrounded by colons (:specifier:sym),
+// while in Mach-O object files (used in MacOS), a postfix notation is used (sym@specifier).
+
+/// AArch64 ELF position-independent addressing:
+///
+///   adrp xN, symbol
+///   add  xN, xN, :lo12:symbol
+///
+/// The :lo12: modifier selects the low 12 bits of the symbol address
+/// and emits an ELF relocation such as R_AARCH64_ADD_ABS_LO12_NC.
+///
+/// Defined by the AArch64 ELF psABI.
+/// See: <https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst#static-miscellaneous-relocations>.
+#[cfg(not(target_vendor = "apple"))]
+macro_rules! sym {
+    ($sym:literal) => {
+        $sym
+    };
+}
+
+#[cfg(not(target_vendor = "apple"))]
+macro_rules! sym_off {
+    ($sym:literal) => {
+        concat!(":lo12:", $sym)
+    };
+}
+
+/// Mach-O ARM64 relocation types:
+///   ARM64_RELOC_PAGE21
+///   ARM64_RELOC_PAGEOFF12
+///
+/// These relocations implement the @PAGE / @PAGEOFF split used by
+/// adrp + add sequences on Apple platforms.
+///
+///   adrp xN, symbol@PAGE      -> ARM64_RELOC_PAGE21
+///   add  xN, xN, symbol@PAGEOFF -> ARM64_RELOC_PAGEOFF12
+///
+/// Relocation types defined by Apple in XNU: <mach-o/arm64/reloc.h>.
+/// See: <https://github.com/apple-oss-distributions/xnu/blob/f6217f891ac0bb64f3d375211650a4c1ff8ca1ea/EXTERNAL_HEADERS/mach-o/arm64/reloc.h>.
+#[cfg(target_vendor = "apple")]
+macro_rules! sym {
+    ($sym:literal) => {
+        concat!($sym, "@PAGE")
+    };
+}
+
+#[cfg(target_vendor = "apple")]
+macro_rules! sym_off {
+    ($sym:literal) => {
+        concat!($sym, "@PAGEOFF")
+    };
+}
+
+/// Emit the required AArch64 sign-extension for signed integer values
+/// smaller than 32 bits, operating in-place on the given register.
+///
+/// Intended for use in naked assembly before `ret` or before a value
+/// is compared/consumed at full register width.
+#[rustfmt::skip]
+macro_rules! sign_extend {
+    (1) => {
+        concat!("sxtb ", reg!(1, 0), ", ", reg!(1, 0))
+    };
+    (2) => {
+        concat!("sxth ", reg!(2, 0), ", ", reg!(2, 0))
+    };
+    (4) => { "" };
+    (8) => { "" };
+    (16) => { "" };
+}
+
 // If supported, perform the requested LSE op and return, or fallthrough.
 macro_rules! try_lse_op {
     ($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => {
         concat!(
-            ".arch_extension lse; ",
-            "adrp    x16, {have_lse}; ",
-            "ldrb    w16, [x16, :lo12:{have_lse}]; ",
-            "cbz     w16, 8f; ",
+            ".arch_extension lse\n",
+            concat!("adrp    x16, ", sym!("{have_lse}"), "\n"),
+            concat!("ldrb    w16, [x16, ", sym_off!("{have_lse}"), "]\n"),
+            "cbz     w16, 8f\n",
             // LSE_OP  s(reg),* [$mem]
-            concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]; ",),
-            "ret; ",
+            concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",),
+            "ret\n",
+            // SXTB s(0), s(0)
+            concat!(sign_extend!($bytes), "\n"),
             "8:"
         )
     };
@@ -195,6 +270,8 @@ macro_rules! compare_and_swap {
                     concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"),
                     "cbnz   w17, 0b",
                     "1:",
+                    // SXTB s(0), s(0)
+                    sign_extend!($bytes),
                     "ret",
                     have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }
@@ -255,6 +332,8 @@ macro_rules! swap {
                     // STXR   w(tmp1), s(tmp0), [x1]
                     concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"),
                     "cbnz   w17, 0b",
+                    // SXTB s(0), s(0)
+                    sign_extend!($bytes),
                     "ret",
                     have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }
@@ -285,6 +364,8 @@ macro_rules! fetch_op {
                     // STXR   w(tmp2), s(tmp1), [x1]
                     concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"),
                     "cbnz  w15, 0b",
+                    // SXTB s(0), s(0)
+                    sign_extend!($bytes),
                     "ret",
                     have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS,
                 }

compiler-builtins/src/lib.rs

@@ -55,7 +55,12 @@ pub mod arm;
 #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
 pub mod aarch64;
 
-#[cfg(all(target_arch = "aarch64", target_feature = "outline-atomics"))]
+// Note that we enable the module on "mangled-names" because that is the default feature
+// in the builtins-test tests. So this is a way of enabling the module during testing.
+#[cfg(all(
+    target_arch = "aarch64",
+    any(target_feature = "outline-atomics", feature = "mangled-names")
+))]
 pub mod aarch64_outline_atomics;
 
 #[cfg(all(