diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
index 602ebc98..920ddb9c 100644
--- a/.github/.cspell/project-dictionary.txt
+++ b/.github/.cspell/project-dictionary.txt
@@ -1,4 +1,3 @@
-addi
amocas
amoswap
amswap
@@ -38,13 +37,11 @@ isync
kuser
LAAL
lclang
-ldapr
ldar
ldarx
ldaxp
ldclrp
ldiapp
-ldrd
ldrex
ldrexd
ldsetp
@@ -70,12 +67,9 @@ memw
mfcr
mfence
mgba
-movd
movi
movlps
movq
-movsd
-movss
mspdebug
msync
muldi
@@ -84,7 +78,6 @@ nostartfiles
opensbi
orrs
partword
-pshufd
pstq
qbsp
quadword
@@ -99,13 +92,11 @@ SCWP
SCWPE
seqz
sete
-shufps
signedness
simavr
simio
slli
sllv
-sltui
srai
sreg
srlv
@@ -115,7 +106,6 @@ stilp
stlxp
stpq
stqcx
-strd
strex
strexd
stwcx
@@ -136,7 +126,6 @@ xadd
xchg
xmegau
xmmword
-xorps
zaamo
zabha
zacas
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0906d267..b4bc45d3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,8 +56,6 @@ jobs:
include:
# ------------------------------------------------------------
# x86_64-unknown-linux-gnu
- - rust: '1.59' # LLVM 13
- target: x86_64-unknown-linux-gnu
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: x86_64-unknown-linux-gnu
# - rust: '1.81' # LLVM 18
@@ -89,9 +87,6 @@ jobs:
os: windows-latest
# ------------------------------------------------------------
# aarch64-unknown-linux-gnu
- - rust: '1.59' # LLVM 13
- target: aarch64-unknown-linux-gnu
- os: ubuntu-24.04-arm
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: aarch64-unknown-linux-gnu
os: ubuntu-24.04-arm
@@ -120,9 +115,6 @@ jobs:
# flags: -Z codegen-backend=cranelift
# ------------------------------------------------------------
# aarch64-apple-darwin
- - rust: '1.59' # LLVM 13
- target: aarch64-apple-darwin
- os: macos-latest
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: aarch64-apple-darwin
os: macos-latest
@@ -148,8 +140,6 @@ jobs:
os: windows-11-arm
# ------------------------------------------------------------
# aarch64_be-unknown-linux-gnu
- - rust: nightly-2021-12-16 # Rust 1.59, LLVM 13 (oldest version we can use stable asm on this target)
- target: aarch64_be-unknown-linux-gnu
- rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: aarch64_be-unknown-linux-gnu
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
@@ -175,8 +165,6 @@ jobs:
os: windows-11-arm
# ------------------------------------------------------------
# armv5te-unknown-linux-gnueabi
- - rust: '1.59' # LLVM 13
- target: armv5te-unknown-linux-gnueabi
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: armv5te-unknown-linux-gnueabi
# - rust: '1.81' # LLVM 18
@@ -191,8 +179,6 @@ jobs:
target: armv5te-unknown-linux-gnueabi
# ------------------------------------------------------------
# arm-unknown-linux-gnueabi
- - rust: '1.59' # LLVM 13
- target: arm-unknown-linux-gnueabi
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: arm-unknown-linux-gnueabi
# - rust: '1.81' # LLVM 18
@@ -208,8 +194,6 @@ jobs:
target: arm-unknown-linux-gnueabi
# ------------------------------------------------------------
# armv7-unknown-linux-gnueabi
- - rust: '1.59' # LLVM 13
- target: armv7-unknown-linux-gnueabi
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: armv7-unknown-linux-gnueabi
# - rust: '1.81' # LLVM 18
@@ -225,9 +209,6 @@ jobs:
target: armv7-unknown-linux-gnueabi
# ------------------------------------------------------------
# armv7-unknown-linux-gnueabihf
- - rust: '1.59' # LLVM 13
- target: armv7-unknown-linux-gnueabihf
- os: ubuntu-24.04-arm
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: armv7-unknown-linux-gnueabihf
os: ubuntu-24.04-arm
@@ -251,9 +232,6 @@ jobs:
os: ubuntu-latest # QEMU on x86_64
# ------------------------------------------------------------
# armeb-unknown-linux-gnueabi
- - rust: nightly-2022-09-16 # Rust 1.65, LLVM 15 (version that this target was added)
- target: armeb-unknown-linux-gnueabi
- os: ubuntu-22.04
- rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: armeb-unknown-linux-gnueabi
os: ubuntu-22.04
@@ -268,8 +246,6 @@ jobs:
os: ubuntu-22.04
# ------------------------------------------------------------
# i586-unknown-linux-gnu
- - rust: '1.59' # LLVM 13
- target: i586-unknown-linux-gnu
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: i586-unknown-linux-gnu
# - rust: '1.81' # LLVM 18
@@ -284,8 +260,6 @@ jobs:
target: i586-unknown-linux-gnu
# ------------------------------------------------------------
# i686-unknown-linux-gnu
- - rust: '1.59' # LLVM 13
- target: i686-unknown-linux-gnu
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: i686-unknown-linux-gnu
# - rust: '1.81' # LLVM 18
@@ -306,8 +280,6 @@ jobs:
target: hexagon-unknown-linux-musl
# ------------------------------------------------------------
# loongarch64-unknown-linux-gnu
- - rust: '1.72' # LLVM 16 (inline asm for loongarch64 has been stabilized in Rust 1.72)
- target: loongarch64-unknown-linux-gnu
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: loongarch64-unknown-linux-gnu
# - rust: '1.81' # LLVM 18
@@ -323,7 +295,7 @@ jobs:
target: loongarch64-unknown-linux-gnu
# ------------------------------------------------------------
# mips-unknown-linux-gnu
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: mips-unknown-linux-gnu
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: mips-unknown-linux-gnu
@@ -333,7 +305,7 @@ jobs:
target: mips-unknown-linux-gnu
# ------------------------------------------------------------
# mipsel-unknown-linux-gnu
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: mipsel-unknown-linux-gnu
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: mipsel-unknown-linux-gnu
@@ -343,7 +315,7 @@ jobs:
target: mipsel-unknown-linux-gnu
# ------------------------------------------------------------
# mips64-unknown-linux-gnuabi64
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: mips64-unknown-linux-gnuabi64
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: mips64-unknown-linux-gnuabi64
@@ -353,7 +325,7 @@ jobs:
target: mips64-unknown-linux-gnuabi64
# ------------------------------------------------------------
# mips64el-unknown-linux-gnuabi64
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: mips64el-unknown-linux-gnuabi64
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: mips64el-unknown-linux-gnuabi64
@@ -403,7 +375,7 @@ jobs:
target: mipsisa64r6el-unknown-linux-gnuabi64
# ------------------------------------------------------------
# powerpc-unknown-linux-gnu
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: powerpc-unknown-linux-gnu
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: powerpc-unknown-linux-gnu
@@ -413,7 +385,7 @@ jobs:
target: powerpc-unknown-linux-gnu
# ------------------------------------------------------------
# powerpc64-unknown-linux-gnu
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: powerpc64-unknown-linux-gnu
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: powerpc64-unknown-linux-gnu
@@ -423,7 +395,7 @@ jobs:
target: powerpc64-unknown-linux-gnu
# ------------------------------------------------------------
# powerpc64le-unknown-linux-gnu
- - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version we can use asm_experimental_arch on this target)
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
target: powerpc64le-unknown-linux-gnu
# - rust: nightly-2024-07-31 # Rust 1.82, LLVM 18
# target: powerpc64le-unknown-linux-gnu
@@ -443,8 +415,6 @@ jobs:
target: riscv32gc-unknown-linux-gnu
# ------------------------------------------------------------
# riscv64gc-unknown-linux-gnu
- - rust: '1.59' # LLVM 13
- target: riscv64gc-unknown-linux-gnu
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: riscv64gc-unknown-linux-gnu
# - rust: '1.81' # LLVM 18
@@ -483,9 +453,6 @@ jobs:
target: sparc64-unknown-linux-gnu
# ------------------------------------------------------------
# thumbv7neon-unknown-linux-gnueabihf
- - rust: '1.59' # LLVM 13
- target: thumbv7neon-unknown-linux-gnueabihf
- os: ubuntu-24.04-arm
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
target: thumbv7neon-unknown-linux-gnueabihf
os: ubuntu-24.04-arm
@@ -676,8 +643,6 @@ jobs:
- run: cargo minimal-versions build --workspace --no-private --detach-path-deps=skip-exact --all-features ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
- run: cargo minimal-versions build --workspace --no-private --detach-path-deps=skip-exact --all-features ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-} --direct
- # -Z direct-minimal-versions requires Cargo 1.70.
- if: (!(startsWith(matrix.rust, '1.5') || startsWith(matrix.rust, '1.6') || startsWith(matrix.rust, 'nightly-2021') || startsWith(matrix.rust, 'nightly-2022') || startsWith(matrix.rust, 'nightly-2023')))
build:
needs: tidy
@@ -686,12 +651,10 @@ jobs:
fail-fast: false
matrix:
include:
- - rust: '1.59'
- rust: '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
- rust: stable
- rust: beta
- - rust: nightly-2023-08-23 # The last nightly version that doesn't support MaybeUninit registers.
- - rust: nightly-2023-08-24 # The oldest nightly version that supports MaybeUninit registers: https://github.com/rust-lang/rust/pull/114790
+ - rust: nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
- rust: nightly
# Check that test suite can be built
- name: nightly, --tests
@@ -715,7 +678,7 @@ jobs:
fail-fast: false
matrix:
rust:
- - '1.64' # LLVM 14
+ - '1.74' # LLVM 17 (oldest version that MaybeUninit register is supported)
- stable
# - beta
- nightly-2023-08-24 # Rust 1.74, LLVM 17 (oldest version that MaybeUninit register is supported)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 23908e1d..00306ffa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,12 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com
## [Unreleased]
+- Increase the minimum supported Rust version (MSRV) from Rust 1.59 to Rust 1.74.
+
+- Remove `AtomicMaybeUninit::const_new` because `AtomicMaybeUninit::new` is now always `const fn` because of the MSRV bump.
+
+- Remove legacy [inefficient](https://github.com/crossbeam-rs/crossbeam/pull/1015#issuecomment-1676549870) implementation for pre-1.74 Rust which doesn't support `MaybeUninit` registers in `asm!`.
+
## [0.3.9] - 2025-09-05
- Update to stabilized [LoongArch32](https://github.com/rust-lang/rust/pull/144402) inline assembly. ([803b062](https://github.com/taiki-e/atomic-maybe-uninit/commit/803b06263c8a3e38596eb48aec88d2dce77d60e6))
diff --git a/Cargo.toml b/Cargo.toml
index a982bfd3..33adca03 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,7 +2,7 @@
name = "atomic-maybe-uninit"
version = "0.3.9" #publish:version
edition = "2021"
-rust-version = "1.59" # For asm!
+rust-version = "1.74" # For MaybeUninit registers in asm!: https://github.com/rust-lang/rust/pull/114790
license = "Apache-2.0 OR MIT"
repository = "https://github.com/taiki-e/atomic-maybe-uninit"
keywords = ["atomic"]
@@ -30,14 +30,13 @@ allowed_external_types = [
doc-scrape-examples = false
[dev-dependencies]
-build-context = "0.1"
-crossbeam-utils = { version = "0.8", git = "https://github.com/crossbeam-rs/crossbeam.git", rev = "17fb841" } # The latest released crossbeam-utils requires Rust 1.60
+crossbeam-utils = "0.8"
fastrand = "2"
paste = "1"
quickcheck = { version = "1", default-features = false, git = "https://github.com/taiki-e/quickcheck.git", rev = "83b1d59" } # https://github.com/BurntSushi/quickcheck/pull/304 + https://github.com/BurntSushi/quickcheck/pull/282 + https://github.com/BurntSushi/quickcheck/pull/296 + f16/f128 support + lower MSRV
[target.'cfg(unix)'.dev-dependencies]
-libc = "=0.2.163" # newer libc requires Rust 1.63
+libc = "0.2"
[lints]
workspace = true
@@ -96,7 +95,7 @@ unexpected_cfgs = { level = "warn", check-cfg = [
] }
unnameable_types = "warn"
unreachable_pub = "warn"
-# unsafe_op_in_unsafe_fn = "warn" # Set at crate-level instead since https://github.com/rust-lang/rust/pull/100081 merged in Rust 1.65 is not available on MSRV
+unsafe_op_in_unsafe_fn = "warn"
[workspace.lints.clippy]
all = "warn" # Downgrade deny-by-default lints
pedantic = "warn"
diff --git a/README.md b/README.md
index 3a50adc7..b1c5b814 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
[](https://crates.io/crates/atomic-maybe-uninit)
[](https://docs.rs/atomic-maybe-uninit)
[](#license)
-[](https://www.rust-lang.org)
+[](https://www.rust-lang.org)
[](https://github.com/taiki-e/atomic-maybe-uninit/actions)
@@ -32,10 +32,10 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch, Arm64EC, s390x, MIPS, P
| riscv32 (+zacas) \[4] | i64,u64 | ✓ | ✓ |
| riscv64 | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓\[1] |
| riscv64 (+zacas) \[4] | i128,u128 | ✓ | ✓ |
-| loongarch64 \[7] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ |
-| loongarch32 \[10] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
-| arm64ec \[8] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
-| s390x \[8] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
+| loongarch64 | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ |
+| loongarch32 \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
+| arm64ec \[7] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
+| s390x \[7] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
| mips / mips32r6 \[9] | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
| mips64 / mips64r6 \[9] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ |
| powerpc \[9] | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
@@ -55,9 +55,8 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch, Arm64EC, s390x, MIPS, P
\[4] Requires `zacas` target feature.
\[5] Requires `quadword-atomics` target feature (enabled by default on powerpc64le).
\[6] Requires `v9` or `leoncasa` target feature (enabled by default on Linux).
-\[7] Requires Rust 1.72+.
-\[8] Requires Rust 1.84+.
-\[10] Requires Rust 1.91+.
+\[7] Requires Rust 1.84+.
+\[8] Requires Rust 1.91+.
\[9] Requires nightly due to `#![feature(asm_experimental_arch)]`.
See also [Atomic operation overview by architecture](https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md) for more information about atomic operations in these architectures.
diff --git a/build.rs b/build.rs
index 85cbafe9..495ece3f 100644
--- a/build.rs
+++ b/build.rs
@@ -34,12 +34,12 @@ fn main() {
// Custom cfgs set by build script. Not public API.
// grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
- "cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_no_asm,atomic_maybe_uninit_no_asm_maybe_uninit,atomic_maybe_uninit_no_cmpxchg,atomic_maybe_uninit_no_cmpxchg8b,atomic_maybe_uninit_no_const_fn_trait_bound,atomic_maybe_uninit_no_const_mut_refs,atomic_maybe_uninit_no_diagnostic_namespace,atomic_maybe_uninit_no_inline_const,atomic_maybe_uninit_no_strict_provenance,atomic_maybe_uninit_no_sync,atomic_maybe_uninit_pre_llvm_20,atomic_maybe_uninit_target_feature,atomic_maybe_uninit_unstable_asm_experimental_arch)"
+ "cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_no_asm,atomic_maybe_uninit_no_cmpxchg,atomic_maybe_uninit_no_cmpxchg8b,atomic_maybe_uninit_no_const_mut_refs,atomic_maybe_uninit_no_diagnostic_namespace,atomic_maybe_uninit_no_inline_const,atomic_maybe_uninit_no_strict_provenance,atomic_maybe_uninit_no_sync,atomic_maybe_uninit_pre_llvm_20,atomic_maybe_uninit_target_feature,atomic_maybe_uninit_unstable_asm_experimental_arch)"
);
// TODO: handle multi-line target_feature_fallback
// grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
- r#"cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_target_feature,values("a","cmpxchg16b","fast-serialization","isa-68020","leoncasa","lse","lse128","lse2","mclass","msync","partword-atomics","quadword-atomics","rcpc","rcpc3","v5te","v6","v7","v8","v8m","v9","x87","zaamo","zabha","zacas","zalrsc"))"#
+ r#"cargo:rustc-check-cfg=cfg(atomic_maybe_uninit_target_feature,values("a","fast-serialization","isa-68020","leoncasa","lse128","lse2","mclass","msync","partword-atomics","quadword-atomics","rcpc3","v5te","v6","v7","v8","v8m","v9","x87","zaamo","zabha","zacas","zalrsc"))"#
);
}
@@ -63,14 +63,6 @@ fn main() {
// stable rustc is used when the build script doesn't run. This is useful
// for non-cargo build systems that don't run the build script.
- // const_fn_trait_bound stabilized in Rust 1.61 (nightly-2022-03-08): https://github.com/rust-lang/rust/pull/93827
- if !version.probe(61, 2022, 3, 7) {
- println!("cargo:rustc-cfg=atomic_maybe_uninit_no_const_fn_trait_bound");
- }
- // https://github.com/rust-lang/rust/pull/114790 merged in nightly-2023-08-24
- if !version.probe(74, 2023, 8, 23) {
- println!("cargo:rustc-cfg=atomic_maybe_uninit_no_asm_maybe_uninit");
- }
// #[diagnostic] stabilized in Rust 1.78 (nightly-2024-03-09): https://github.com/rust-lang/rust/pull/119888
if !version.probe(78, 2024, 3, 8) {
println!("cargo:rustc-cfg=atomic_maybe_uninit_no_diagnostic_namespace");
@@ -93,12 +85,6 @@ fn main() {
}
match target_arch {
- "loongarch64" => {
- // asm! on LoongArch64 stabilized in Rust 1.72
- if version.minor < 72 {
- println!("cargo:rustc-cfg=atomic_maybe_uninit_no_asm");
- }
- }
"arm64ec" | "s390x" => {
// asm! on Arm64EC and s390x stabilized in Rust 1.84 (nightly-2024-11-11): https://github.com/rust-lang/rust/pull/131781, https://github.com/rust-lang/rust/pull/131258
if !version.probe(84, 2024, 11, 10) {
@@ -186,48 +172,18 @@ fn main() {
println!("cargo:rustc-cfg=atomic_maybe_uninit_no_cmpxchg8b");
}
}
- "x86_64" => {
- // cmpxchg16b_target_feature stabilized in Rust 1.69. (for arch_legacy)
- if needs_target_feature_fallback(&version, Some(69)) {
- // x86_64 Apple targets always support CMPXCHG16B:
- // https://github.com/rust-lang/rust/blob/1.68.0/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs#L8
- // https://github.com/rust-lang/rust/blob/1.68.0/compiler/rustc_target/src/spec/apple_base.rs#L69-L70
- // (Since Rust 1.78, Windows (except Windows 7) targets also enable CMPXCHG16B, but
- // this branch is only used on pre-1.69 that cmpxchg16b_target_feature is unstable.)
- // Script to get builtin targets that support CMPXCHG16B by default:
- // $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "x86_64" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"cmpxchg16b"' && printf '%s\n' "${target}"; done)
- let is_apple = env::var("CARGO_CFG_TARGET_VENDOR").unwrap_or_default() == "apple";
- let cmpxchg16b = is_apple;
- // LLVM recognizes this also as cx16 target feature: https://godbolt.org/z/KM3jz616j
- // However, it is unlikely that rustc will support that name, so we ignore it.
- target_feature_fallback("cmpxchg16b", cmpxchg16b);
- }
- }
"aarch64" | "arm64ec" => {
// target_feature "lse2"/"lse128"/"rcpc3" is unstable and available on rustc side since nightly-2024-08-30: https://github.com/rust-lang/rust/pull/128192
if !version.probe(82, 2024, 8, 29) || needs_target_feature_fallback(&version, None) {
- // FEAT_LSE2 doesn't imply FEAT_LSE. FEAT_LSE128 implies FEAT_LSE but not FEAT_LSE2. FEAT_LRCPC3 implies FEAT_LRCPC.
- // AArch64 macOS always supports FEAT_LSE/FEAT_LSE2/FEAT_LRCPC because M1 is Armv8.4 with all features of Armv8.5 except FEAT_BTI:
+ // AArch64 macOS always supports FEAT_LSE2 because M1 is Armv8.4 with all features of Armv8.5 except FEAT_BTI:
// https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0/llvm/lib/Target/AArch64/AArch64Processors.td#L1289
// https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0/llvm/lib/Target/AArch64/AArch64Processors.td#L941
- // Script to get builtin targets that support FEAT_LSE/FEAT_LSE2/FEAT_LRCPC by default:
- // $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "aarch64" or .value.arch == "arm64ec" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"lse"' && printf '%s\n' "${target}"; done)
+ // Script to get builtin targets that support FEAT_LSE2 by default:
// $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "aarch64" or .value.arch == "arm64ec" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"lse2"' && printf '%s\n' "${target}"; done)
- // $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == "aarch64" or .value.arch == "arm64ec" then .key else empty end'); do rustc --print cfg --target "${target}" | grep -Fq '"rcpc"' && printf '%s\n' "${target}"; done)
let is_macos = target_os == "macos";
- let mut lse = is_macos;
- let mut rcpc = is_macos;
target_feature_fallback("lse2", is_macos);
- // LLVM supports FEAT_LRCPC3 and FEAT_LSE128 on LLVM 16+:
- // https://github.com/llvm/llvm-project/commit/a6aaa969f7caec58a994142f8d855861cf3a1463
- // https://github.com/llvm/llvm-project/commit/7fea6f2e0e606e5339c3359568f680eaf64aa306
- lse |= target_feature_fallback("lse128", false);
- rcpc |= target_feature_fallback("rcpc3", false);
- // aarch64_target_feature stabilized in Rust 1.61. (for arch_legacy)
- if needs_target_feature_fallback(&version, Some(61)) {
- target_feature_fallback("lse", lse);
- target_feature_fallback("rcpc", rcpc);
- }
+ target_feature_fallback("lse128", false);
+ target_feature_fallback("rcpc3", false);
}
}
"arm" => {
diff --git a/src/arch_legacy/aarch64.rs b/src/arch_legacy/aarch64.rs
deleted file mode 100644
index 9220ca67..00000000
--- a/src/arch_legacy/aarch64.rs
+++ /dev/null
@@ -1,898 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-AArch64
-
-See arch/aarch64.rs for references and notes.
-
-Generated asm:
-- aarch64 https://godbolt.org/z/6TKofhrbb
-- aarch64 msvc https://godbolt.org/z/5GzETjcE7
-- aarch64 (+lse) https://godbolt.org/z/7jK5vej7b
-- aarch64 msvc (+lse) https://godbolt.org/z/896zWazdW
-- aarch64 (+lse,+lse2) https://godbolt.org/z/66cMd4Ys6
-- aarch64 (+lse,+lse2,+rcpc3) https://godbolt.org/z/ojbaYn9Kf
-- aarch64 (+rcpc) https://godbolt.org/z/4ahePW8TK
-- aarch64 (+lse2,+lse128) https://godbolt.org/z/joMq5vv1h
-- aarch64 (+lse2,+lse128,+rcpc3) https://godbolt.org/z/WdbsccKcz
-*/
-
-use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering};
-
-use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
-
-macro_rules! atomic_rmw {
- ($op:ident, $order:ident) => {
- atomic_rmw!($op, $order, write = $order)
- };
- ($op:ident, $order:ident, write = $write:ident) => {
- match $order {
- Ordering::Relaxed => $op!("", "", ""),
- Ordering::Acquire => $op!("a", "", ""),
- Ordering::Release => $op!("", "l", ""),
- Ordering::AcqRel => $op!("a", "l", ""),
- // In MSVC environments, SeqCst stores/writes needs fences after writes.
- // https://reviews.llvm.org/D141748
- #[cfg(target_env = "msvc")]
- Ordering::SeqCst if $write == Ordering::SeqCst => $op!("a", "l", "dmb ish"),
- // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
- Ordering::SeqCst => $op!("a", "l", ""),
- _ => unreachable!(),
- }
- };
-}
-
-macro_rules! atomic {
- ($ty:ident, $asm_suffix:tt, $val_modifier:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($acquire:tt) => {
- asm!(
- // (atomic) load from src to tmp
- concat!("ld", $acquire, "r", $asm_suffix, " {tmp", $val_modifier, "}, [{src}]"),
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp", $val_modifier, "}, [{out}]"),
- src = in(reg) ptr_reg!(src),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!(""),
- #[cfg(any(target_feature = "rcpc", atomic_maybe_uninit_target_feature = "rcpc"))]
- Ordering::Acquire => {
- // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC.
- asm!(
- // (atomic) load from src to tmp
- concat!("ldapr", $asm_suffix, " {tmp", $val_modifier, "}, [{src}]"),
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp", $val_modifier, "}, [{out}]"),
- src = in(reg) ptr_reg!(src),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- );
- }
- #[cfg(not(any(target_feature = "rcpc", atomic_maybe_uninit_target_feature = "rcpc")))]
- Ordering::Acquire => atomic_load!("a"),
- Ordering::SeqCst => atomic_load!("a"),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store {
- ($release:tt, $fence:tt) => {
- asm!(
- // load from val to tmp
- concat!("ldr", $asm_suffix, " {tmp", $val_modifier, "}, [{val}]"),
- // (atomic) store tmp to dst
- concat!("st", $release, "r", $asm_suffix, " {tmp", $val_modifier, "}, [{dst}]"),
- $fence,
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!("", ""),
- Ordering::Release => atomic_store!("l", ""),
- // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
- #[cfg(not(target_env = "msvc"))]
- Ordering::SeqCst => atomic_store!("l", ""),
- // In MSVC environments, SeqCst stores/writes needs fences after writes.
- // https://reviews.llvm.org/D141748
- #[cfg(target_env = "msvc")]
- Ordering::SeqCst => atomic_store!("l", "dmb ish"),
- _ => unreachable!(),
- }
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- #[cfg(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse"))]
- macro_rules! swap {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from val to tmp
- concat!("ldr", $asm_suffix, " {tmp", $val_modifier, "}, [{val}]"),
- // (atomic) swap
- // Refs:
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWP--SWPA--SWPAL--SWPL--Swap-word-or-doubleword-in-memory-
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWPB--SWPAB--SWPALB--SWPLB--Swap-byte-in-memory-
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWPH--SWPAH--SWPALH--SWPLH--Swap-halfword-in-memory-
- concat!("swp", $acquire, $release, $asm_suffix, " {tmp", $val_modifier, "}, {tmp", $val_modifier, "}, [{dst}]"),
- $fence,
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp", $val_modifier, "}, [{out}]"),
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- #[cfg(not(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse")))]
- macro_rules! swap {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from val to val_tmp
- concat!("ldr", $asm_suffix, " {val_tmp", $val_modifier, "}, [{val}]"),
- // (atomic) swap (LL/SC loop)
- "2:",
- // load from dst to out_tmp
- concat!("ld", $acquire, "xr", $asm_suffix, " {out_tmp", $val_modifier, "}, [{dst}]"),
- // try to store val to dst
- concat!("st", $release, "xr", $asm_suffix, " {r:w}, {val_tmp", $val_modifier, "}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {r:w}, 2b",
- $fence,
- // store out_tmp to out
- concat!("str", $asm_suffix, " {out_tmp", $val_modifier, "}, [{out}]"),
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- val_tmp = out(reg) _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- out_tmp = out(reg) _,
- r = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw!(swap, order);
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- #[cfg(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse"))]
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt, $fence:tt) => {{
- asm!(
- // load from old/new to old_tmp/new_tmp
- concat!("ldr", $asm_suffix, " {old_tmp", $val_modifier, "}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new_tmp", $val_modifier, "}, [{new}]"),
- // cas writes the current value to the first register,
- // so copy the `old`'s value for later comparison.
- concat!("mov {out_tmp", $val_modifier, "}, {old_tmp", $val_modifier, "}"),
- // (atomic) CAS
- // Refs:
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CAS--CASA--CASAL--CASL--Compare-and-swap-word-or-doubleword-in-memory-
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASB--CASAB--CASALB--CASLB--Compare-and-swap-byte-in-memory-
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASH--CASAH--CASALH--CASLH--Compare-and-swap-halfword-in-memory-
- concat!("cas", $acquire, $release, $asm_suffix, " {out_tmp", $val_modifier, "}, {new_tmp", $val_modifier, "}, [{dst}]"),
- $fence,
- concat!("cmp {out_tmp", $val_modifier, "}, {old_tmp", $val_modifier, "}"),
- // store out_tmp to out
- concat!("str", $asm_suffix, " {out_tmp", $val_modifier, "}, [{out}]"),
- "cset {r:w}, eq",
- dst = inout(reg) ptr_reg!(dst) => _,
- old = in(reg) ptr_reg!(old),
- old_tmp = out(reg) _,
- new = in(reg) ptr_reg!(new),
- new_tmp = out(reg) _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- out_tmp = out(reg) _,
- r = lateout(reg) r,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }};
- }
- #[cfg(not(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse")))]
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt, $fence:tt) => {{
- asm!(
- // load from old/new to old_tmp/new_tmp
- concat!("ldr", $asm_suffix, " {new_tmp", $val_modifier, "}, [{new}]"),
- concat!("ldr", $asm_suffix, " {old_tmp", $val_modifier, "}, [{old}]"),
- // (atomic) CAS (LL/SC loop)
- "2:",
- concat!("ld", $acquire, "xr", $asm_suffix, " {out_tmp", $val_modifier, "}, [{dst}]"),
- concat!("cmp {out_tmp", $val_modifier, "}, {old_tmp", $val_modifier, "}"),
- "b.ne 3f", // jump if compare failed
- concat!("st", $release, "xr", $asm_suffix, " {r:w}, {new_tmp", $val_modifier, "}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {r:w}, 2b", // continue loop if store failed
- $fence,
- "b 4f",
- "3:",
- "mov {r:w}, #1", // mark as failed
- "clrex",
- "4:",
- // store out_tmp to out
- concat!("str", $asm_suffix, " {out_tmp", $val_modifier, "}, [{out}]"),
- dst = inout(reg) ptr_reg!(dst) => _,
- old = in(reg) ptr_reg!(old),
- old_tmp = out(reg) _,
- new = in(reg) ptr_reg!(new),
- new_tmp = out(reg) _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- out_tmp = out(reg) _,
- r = lateout(reg) r,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }};
- }
- atomic_rmw!(cmpxchg, order, write = success)
- }
- }
- #[cfg(not(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse")))]
- #[inline]
- unsafe fn atomic_compare_exchange_weak(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let r: i32;
- macro_rules! cmpxchg_weak {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from old/new to old_tmp/new_tmp
- concat!("ldr", $asm_suffix, " {new_tmp", $val_modifier, "}, [{new}]"),
- concat!("ldr", $asm_suffix, " {old_tmp", $val_modifier, "}, [{old}]"),
- // (atomic) CAS
- concat!("ld", $acquire, "xr", $asm_suffix, " {out_tmp", $val_modifier, "}, [{dst}]"),
- concat!("cmp {out_tmp", $val_modifier, "}, {old_tmp", $val_modifier, "}"),
- "b.ne 3f",
- concat!("st", $release, "xr", $asm_suffix, " {r:w}, {new_tmp", $val_modifier, "}, [{dst}]"),
- // TODO: emit fence only when the above sc succeed?
- // // 0 if the store was successful, 1 if no store was performed
- // "cbnz {r:w}, 4f",
- $fence,
- "b 4f",
- "3:",
- "mov {r:w}, #1",
- "clrex",
- "4:",
- // store out_tmp to out
- concat!("str", $asm_suffix, " {out_tmp", $val_modifier, "}, [{out}]"),
- dst = inout(reg) ptr_reg!(dst) => _,
- old = in(reg) ptr_reg!(old),
- old_tmp = out(reg) _,
- new = in(reg) ptr_reg!(new),
- new_tmp = out(reg) _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- out_tmp = out(reg) _,
- r = lateout(reg) r,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(cmpxchg_weak, order, write = success);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic!(i8, "b", ":w");
-atomic!(u8, "b", ":w");
-atomic!(i16, "h", ":w");
-atomic!(u16, "h", ":w");
-atomic!(i32, "", ":w");
-atomic!(u32, "", ":w");
-atomic!(i64, "", "");
-atomic!(u64, "", "");
-#[cfg(target_pointer_width = "32")]
-atomic!(isize, "", ":w");
-#[cfg(target_pointer_width = "32")]
-atomic!(usize, "", ":w");
-#[cfg(target_pointer_width = "64")]
-atomic!(isize, "", "");
-#[cfg(target_pointer_width = "64")]
-atomic!(usize, "", "");
-
-// There are a few ways to implement 128-bit atomic operations in AArch64.
-//
-// - LDXP/STXP loop (DW LL/SC)
-// - CASP (DWCAS) added as Armv8.1 FEAT_LSE (optional from Armv8.0, mandatory from Armv8.1)
-// - LDP/STP (DW load/store) if Armv8.4 FEAT_LSE2 (optional from Armv8.2, mandatory from Armv8.4) is available
-// - LDIAPP/STILP (DW acquire-load/release-store) added as Armv8.9 FEAT_LRCPC3 (optional from Armv8.2) (if FEAT_LSE2 is also available)
-// - LDCLRP/LDSETP/SWPP (DW RMW) added as Armv9.4 FEAT_LSE128 (optional from Armv9.3)
-//
-// If FEAT_LSE is available at compile-time, we use CASP for load/CAS. Otherwise, use LDXP/STXP loop.
-// If FEAT_LSE2 is available at compile-time, we use LDP/STP for load/store.
-// If FEAT_LSE128 is available at compile-time, we use SWPP for swap/{release,seqcst}-store.
-// If FEAT_LSE2 and FEAT_LRCPC3 are available at compile-time, we use LDIAPP/STILP for acquire-load/release-store.
-//
-// Note: FEAT_LSE2 doesn't imply FEAT_LSE. FEAT_LSE128 implies FEAT_LSE but not FEAT_LSE2.
-//
-// Refs:
-// - LDXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDXP--Load-exclusive-pair-of-registers-
-// - LDAXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDAXP--Load-acquire-exclusive-pair-of-registers-
-// - STXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STXP--Store-exclusive-pair-of-registers-
-// - STLXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STLXP--Store-release-exclusive-pair-of-registers-
-//
-// Note: Load-Exclusive pair (by itself) does not guarantee atomicity; to complete an atomic
-// operation (even load/store), a corresponding Store-Exclusive pair must succeed.
-// See Arm Architecture Reference Manual for A-profile architecture
-// Section B2.2.1 "Requirements for single-copy atomicity", and
-// Section B2.9 "Synchronization and semaphores" for more.
-macro_rules! atomic128 {
- ($ty:ident) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- #[cfg(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2"))]
- // SAFETY: the caller must guarantee that `dst` is valid for reads,
- // 16-byte aligned, that there are no concurrent non-atomic operations.
- // the above cfg guarantee that the CPU supports FEAT_LSE2.
- //
- // Refs:
- // - LDP https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDP--Load-pair-of-registers-
- // - LDIAPP https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDIAPP--Load-Acquire-RCpc-ordered-pair-of-registers-
- unsafe {
- macro_rules! atomic_load_relaxed {
- ($acquire:tt) => {
- asm!(
- // (atomic) load from src to tmp pair
- "ldp {tmp_lo}, {tmp_hi}, [{src}]",
- $acquire,
- // store tmp pair to out
- "stp {tmp_lo}, {tmp_hi}, [{out}]",
- src = in(reg) ptr_reg!(src),
- out = in(reg) ptr_reg!(out_ptr),
- tmp_hi = out(reg) _,
- tmp_lo = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load_relaxed!(""),
- #[cfg(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3"))]
- Ordering::Acquire => {
- // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC3.
- // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDIAPP--Load-Acquire-RCpc-ordered-pair-of-registers-
- asm!(
- // (atomic) load from src to tmp pair
- "ldiapp {tmp_lo}, {tmp_hi}, [{src}]",
- // store tmp pair to out
- "stp {tmp_lo}, {tmp_hi}, [{out}]",
- src = in(reg) ptr_reg!(src),
- out = in(reg) ptr_reg!(out_ptr),
- tmp_hi = out(reg) _,
- tmp_lo = out(reg) _,
- options(nostack, preserves_flags),
- );
- }
- #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
- Ordering::Acquire => atomic_load_relaxed!("dmb ishld"),
- Ordering::SeqCst => {
- asm!(
- // ldar (or dmb ishld) is required to prevent reordering with preceding stlxp.
- // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891
- "ldar {tmp}, [{src}]",
- // (atomic) load from src to tmp pair
- "ldp {tmp_lo}, {tmp_hi}, [{src}]",
- "dmb ishld",
- // store tmp pair to out
- "stp {tmp_lo}, {tmp_hi}, [{out}]",
- src = in(reg) ptr_reg!(src),
- out = in(reg) ptr_reg!(out_ptr),
- tmp_hi = out(reg) _,
- tmp_lo = out(reg) _,
- tmp = out(reg) _,
- options(nostack, preserves_flags),
- );
- },
- _ => unreachable!(),
- }
- }
- #[cfg(not(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2")))]
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- #[cfg(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse"))]
- macro_rules! atomic_load {
- ($acquire:tt, $release:tt) => {
- asm!(
- // (atomic) load (CAS)
- // Refs:
- // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASP--CASPA--CASPAL--CASPL--Compare-and-swap-pair-of-words-or-doublewords-in-memory-
- // - https://github.com/taiki-e/portable-atomic/pull/20
- concat!("casp", $acquire, $release, " x2, x3, x2, x3, [{src}]"),
- // store out pair to out
- "stp x2, x3, [{out}]",
- src = in(reg) ptr_reg!(src),
- out = in(reg) ptr_reg!(out_ptr),
- // must be allocated to even/odd register pair
- inout("x2") 0_u64 => _, // out_lo
- inout("x3") 0_u64 => _, // out_lo
- options(nostack, preserves_flags),
- )
- };
- }
- #[cfg(not(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse")))]
- macro_rules! atomic_load {
- ($acquire:tt, $release:tt) => {
- asm!(
- // (atomic) load from src to tmp pair
- "2:",
- // load from src to tmp pair
- concat!("ld", $acquire, "xp {tmp_lo}, {tmp_hi}, [{src}]"),
- // store tmp pair to src
- concat!("st", $release, "xp {r:w}, {tmp_lo}, {tmp_hi}, [{src}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {r:w}, 2b",
- // store tmp pair to out
- "stp {tmp_lo}, {tmp_hi}, [{out}]",
- src = in(reg) ptr_reg!(src),
- out = in(reg) ptr_reg!(out_ptr),
- tmp_hi = out(reg) _,
- tmp_lo = out(reg) _,
- r = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!("", ""),
- Ordering::Acquire => atomic_load!("a", ""),
- Ordering::SeqCst => atomic_load!("a", "l"),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- #[cfg(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2"))]
- // SAFETY: the caller must guarantee that `dst` is valid for writes,
- // 16-byte aligned, that there are no concurrent non-atomic operations.
- // the above cfg guarantee that the CPU supports FEAT_LSE2.
- //
- // Refs:
- // - STP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STP--Store-pair-of-registers-
- // - STILP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STILP--Store-release-ordered-pair-of-registers-
- unsafe {
- macro_rules! atomic_store {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from val to val pair
- "ldp {val_lo}, {val_hi}, [{val}]",
- // (atomic) store val pair to dst
- $release,
- "stp {val_lo}, {val_hi}, [{dst}]",
- $acquire,
- dst = in(reg) ptr_reg!(dst),
- val = in(reg) ptr_reg!(val),
- val_hi = out(reg) _,
- val_lo = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- // Use swpp if stp requires fences.
- // https://reviews.llvm.org/D143506
- #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
- macro_rules! atomic_store_swpp {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from val to val pair
- "ldp {val_lo}, {val_hi}, [{val}]",
- // (atomic) swap
- concat!("swpp", $acquire, $release, " {val_lo}, {val_hi}, [{dst}]"),
- $fence,
- dst = in(reg) ptr_reg!(dst),
- val = in(reg) ptr_reg!(val),
- val_hi = out(reg) _,
- val_lo = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!("", ""),
- #[cfg(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3"))]
- Ordering::Release => {
- // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC3.
- // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STILP--Store-release-ordered-pair-of-registers-
- asm!(
- // load from val to val pair
- "ldp {val_lo}, {val_hi}, [{val}]",
- // (atomic) store val pair to dst
- "stilp {val_lo}, {val_hi}, [{dst}]",
- dst = in(reg) ptr_reg!(dst),
- val = in(reg) ptr_reg!(val),
- val_hi = out(reg) _,
- val_lo = out(reg) _,
- options(nostack, preserves_flags),
- );
- }
- #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
- #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
- Ordering::Release => atomic_rmw!(atomic_store_swpp, order),
- #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
- #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
- Ordering::Release => atomic_store!("", "dmb ish"),
- #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
- Ordering::SeqCst => atomic_rmw!(atomic_store_swpp, order),
- #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
- Ordering::SeqCst => atomic_store!("dmb ish", "dmb ish"),
- _ => unreachable!(),
- }
- }
- #[cfg(not(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2")))]
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! store {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from val to val pair
- "ldp {val_lo}, {val_hi}, [{val}]",
- // (atomic) store val pair to dst (LL/SC loop)
- "2:",
- // load from dst to xzr/tmp pair
- concat!("ld", $acquire, "xp xzr, {tmp}, [{dst}]"),
- // try to store val pair to dst
- concat!("st", $release, "xp {tmp:w}, {val_lo}, {val_hi}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {tmp:w}, 2b",
- $fence,
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- val_hi = out(reg) _,
- val_lo = out(reg) _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw!(store, order);
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
- macro_rules! swap {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from val to val pair
- "ldp {val_lo}, {val_hi}, [{val}]",
- // (atomic) swap
- concat!("swpp", $acquire, $release, " {val_lo}, {val_hi}, [{dst}]"),
- $fence,
- // store out pair to out
- "stp {val_lo}, {val_hi}, [{out}]",
- dst = in(reg) ptr_reg!(dst),
- val = in(reg) ptr_reg!(val),
- out = in(reg) ptr_reg!(out_ptr),
- val_hi = out(reg) _,
- val_lo = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
- macro_rules! swap {
- ($acquire:tt, $release:tt, $fence:tt) => {
- asm!(
- // load from val to val pair
- "ldp {val_lo}, {val_hi}, [{val}]",
- // (atomic) swap (LL/SC loop)
- "2:",
- // load from dst to out pair
- concat!("ld", $acquire, "xp {out_lo}, {out_hi}, [{dst}]"),
- // try to store val pair to dst
- concat!("st", $release, "xp {r:w}, {val_lo}, {val_hi}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {r:w}, 2b",
- $fence,
- // store out pair to out
- "stp {out_lo}, {out_hi}, [{out}]",
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- val_hi = out(reg) _,
- val_lo = out(reg) _,
- out_hi = out(reg) _,
- out_lo = out(reg) _,
- r = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw!(swap, order);
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- #[cfg(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse"))]
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt, $fence:tt) => {{
- asm!(
- // load from old/new to old/new pairs
- "ldp {old_lo}, {old_hi}, [{old}]",
- "ldp x4, x5, [{new}]",
- // casp writes the current value to the first register pair,
- // so copy the `old`'s value for later comparison.
- "mov x8, {old_lo}",
- "mov x9, {old_hi}",
- // (atomic) CAS
- // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASP--CASPA--CASPAL--CASPL--Compare-and-swap-pair-of-words-or-doublewords-in-memory-
- concat!("casp", $acquire, $release, " x8, x9, x4, x5, [{dst}]"),
- $fence,
- // compare old pair and out pair
- "cmp x8, {old_lo}",
- "ccmp x9, {old_hi}, #0, eq",
- "cset {r:w}, eq",
- // store out pair to out
- "stp x8, x9, [{out}]",
- dst = in(reg) ptr_reg!(dst),
- old = in(reg) ptr_reg!(old),
- new = in(reg) ptr_reg!(new),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- old_lo = out(reg) _,
- old_hi = out(reg) _,
- r = lateout(reg) r,
- // new pair - must be allocated to even/odd register pair
- out("x4") _, // new_lo
- out("x5") _, // new_hi
- // out pair - must be allocated to even/odd register pair
- out("x8") _, // out_lo
- out("x9") _, // out_hi
- // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }};
- }
- #[cfg(not(any(target_feature = "lse", atomic_maybe_uninit_target_feature = "lse")))]
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt, $fence:tt) => {{
- asm!(
- // load from old/new to old/new pair
- "ldp {new_lo}, {new_hi}, [{new}]",
- "ldp {old_lo}, {old_hi}, [{old}]",
- // (atomic) CAS (LL/SC loop)
- "2:",
- concat!("ld", $acquire, "xp {out_lo}, {out_hi}, [{dst}]"),
- "cmp {out_lo}, {old_lo}",
- "cset {r:w}, ne",
- "cmp {out_hi}, {old_hi}",
- "cinc {r:w}, {r:w}, ne",
- "cbz {r:w}, 3f", // jump if compare succeed
- concat!("st", $release, "xp {r:w}, {out_lo}, {out_hi}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {r:w}, 2b", // continue loop if store failed
- "mov {r:w}, #1", // mark as failed
- "b 4f",
- "3:",
- concat!("st", $release, "xp {r:w}, {new_lo}, {new_hi}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cbnz {r:w}, 2b", // continue loop if store failed
- "4:",
- $fence,
- // store out_tmp to out
- "stp {out_lo}, {out_hi}, [{out}]",
- dst = inout(reg) ptr_reg!(dst) => _,
- old = in(reg) ptr_reg!(old),
- old_hi = out(reg) _,
- old_lo = out(reg) _,
- new = in(reg) ptr_reg!(new),
- new_hi = out(reg) _,
- new_lo = out(reg) _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- out_hi = out(reg) _,
- out_lo = out(reg) _,
- r = lateout(reg) r,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }};
- }
- atomic_rmw!(cmpxchg, order, write = success)
- }
- }
- }
- };
-}
-
-atomic128!(i128);
-atomic128!(u128);
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
diff --git a/src/arch_legacy/arm.rs b/src/arch_legacy/arm.rs
deleted file mode 100644
index edb77aa7..00000000
--- a/src/arch_legacy/arm.rs
+++ /dev/null
@@ -1,1207 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-Armv6 and Armv7
-
-See arch/arm.rs for references and notes.
-
-Generated asm:
-- armv7-a https://godbolt.org/z/P93x9TjWs
-- armv7-m https://godbolt.org/z/WozEfbMbx
-- armv6 https://godbolt.org/z/T5M337jYK
-- armv6-m https://godbolt.org/z/q88qPah4W
-*/
-
-use core::{mem::MaybeUninit, sync::atomic::Ordering};
-
-#[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-))]
-use crate::raw::{AtomicCompareExchange, AtomicSwap};
-use crate::raw::{AtomicLoad, AtomicStore};
-
-#[cfg(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"))]
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-macro_rules! dmb {
- () => {
- "dmb ish"
- };
-}
-// Only a full system barrier exists in the M-class architectures.
-#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
-macro_rules! dmb {
- () => {
- "dmb sy"
- };
-}
-// Armv6 does not support `dmb`, so use use special instruction equivalent to a DMB.
-//
-// Refs:
-// - https://reviews.llvm.org/D5386
-// - https://developer.arm.com/documentation/ddi0360/f/control-coprocessor-cp15/register-descriptions/c7--cache-operations-register
-#[cfg(not(all(
- any(target_os = "linux", target_os = "android"),
- not(atomic_maybe_uninit_use_cp15_barrier),
-)))]
-#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-macro_rules! dmb {
- () => {
- "mcr p15, #0, r0, c7, c10, #5"
- };
-}
-// We prefer __kuser_memory_barrier over cp15_barrier because cp15_barrier is
-// trapped and emulated by default on Linux/Android with Armv8+ (or Armv7+?).
-// https://github.com/rust-lang/rust/issues/60605
-#[cfg(all(
- any(target_os = "linux", target_os = "android"),
- not(atomic_maybe_uninit_use_cp15_barrier),
-))]
-#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-macro_rules! dmb {
- () => {
- "blx r0"
- };
-}
-
-#[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-))]
-#[cfg(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"))]
-macro_rules! clrex {
- () => {
- "clrex"
- };
-}
-#[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-))]
-#[cfg(not(any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7")))]
-macro_rules! clrex {
- () => {
- ""
- };
-}
-
-macro_rules! asm_no_dmb {
- (options($($options:tt)*), $($asm:tt)*) => {
- core::arch::asm!(
- $($asm)*
- options($($options)*),
- )
- };
-}
-#[cfg(any(
- target_feature = "v7",
- atomic_maybe_uninit_target_feature = "v7",
- target_feature = "mclass",
- atomic_maybe_uninit_target_feature = "mclass",
-))]
-macro_rules! asm_use_dmb {
- (options($($options:tt)*), $($asm:tt)*) => {
- core::arch::asm!(
- $($asm)*
- options($($options)*),
- )
- };
-}
-#[cfg(not(all(
- any(target_os = "linux", target_os = "android"),
- not(atomic_maybe_uninit_use_cp15_barrier),
-)))]
-#[cfg(not(any(
- target_feature = "v7",
- atomic_maybe_uninit_target_feature = "v7",
- target_feature = "mclass",
- atomic_maybe_uninit_target_feature = "mclass",
-)))]
-macro_rules! asm_use_dmb {
- (options($($options:tt)*), $($asm:tt)*) => {
- // In this case, dmb! calls `mcr p15, 0, , c7, c10, 5`, and the value in the Rd register should be zero (SBZ).
- core::arch::asm!(
- $($asm)*
- inout("r0") 0_u32 => _,
- options($($options)*),
- )
- };
-}
-#[cfg(all(
- any(target_os = "linux", target_os = "android"),
- not(atomic_maybe_uninit_use_cp15_barrier),
-))]
-#[cfg(not(any(
- target_feature = "v7",
- atomic_maybe_uninit_target_feature = "v7",
- target_feature = "mclass",
- atomic_maybe_uninit_target_feature = "mclass",
-)))]
-macro_rules! asm_use_dmb {
- (options($($options:tt)*), $($asm:tt)*) => {
- // In this case, dmb! calls __kuser_memory_barrier.
- core::arch::asm!(
- $($asm)*
- // __kuser_memory_barrier (see also arm_linux.rs)
- // https://github.com/torvalds/linux/blob/v6.16/Documentation/arch/arm/kernel_user_helpers.rst
- inout("r0") 0xFFFF0FA0_usize => _,
- out("lr") _,
- options($($options)*),
- )
- };
-}
-
-macro_rules! atomic {
- ($ty:ident, $asm_suffix:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($asm:ident, $acquire:expr) => {
- $asm!(
- options(nostack, preserves_flags),
- // (atomic) load from src to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{src}]"),
- $acquire, // acquire fence
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- src = in(reg) src,
- out = inout(reg) out_ptr => _,
- tmp = lateout(reg) _,
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!(asm_no_dmb, ""),
- // Acquire and SeqCst loads are equivalent.
- Ordering::Acquire | Ordering::SeqCst => atomic_load!(asm_use_dmb, dmb!()),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store {
- ($asm:ident, $acquire:expr, $release:expr) => {
- $asm!(
- options(nostack, preserves_flags),
- // load from val to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{val}]"),
- // (atomic) store tmp to dst
- $release, // release fence
- concat!("str", $asm_suffix, " {tmp}, [{dst}]"),
- $acquire, // acquire fence
- dst = inout(reg) dst => _,
- val = in(reg) val,
- tmp = lateout(reg) _,
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!(asm_no_dmb, "", ""),
- Ordering::Release => atomic_store!(asm_use_dmb, "", dmb!()),
- Ordering::SeqCst => atomic_store!(asm_use_dmb, dmb!(), dmb!()),
- _ => unreachable!(),
- }
- }
- }
- }
- #[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
- ))]
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_swap {
- ($asm:ident, $acquire:expr, $release:expr) => {
- $asm!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from val (ptr) to val (val)
- concat!("ldr", $asm_suffix, " {val}, [{val}]"),
- // (atomic) swap (LL/SC loop)
- $release, // release fence
- "2:",
- // load from dst to tmp
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- // try to store val to dst
- concat!("strex", $asm_suffix, " {r}, {val}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b",
- $acquire, // acquire fence
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- val = inout(reg) val => _,
- out = in(reg) out_ptr,
- r = out(reg) _,
- tmp = out(reg) _,
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_swap!(asm_no_dmb, "", ""),
- Ordering::Acquire => atomic_swap!(asm_use_dmb, dmb!(), ""),
- Ordering::Release => atomic_swap!(asm_use_dmb, "", dmb!()),
- // AcqRel and SeqCst swaps are equivalent.
- Ordering::AcqRel | Ordering::SeqCst => {
- atomic_swap!(asm_use_dmb, dmb!(), dmb!());
- }
- _ => unreachable!(),
- }
- }
- out
- }
- }
- #[rustfmt::skip]
- #[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
- ))]
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- use core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst};
- let mut r: i32;
- macro_rules! cmpxchg_store_relaxed {
- ($asm:ident, $acquire_success:expr, $acquire_failure:expr) => {
- $asm!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- // (atomic) CAS (LL/SC loop)
- "2:",
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- concat!("strex", $asm_suffix, " {r}, {new}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b", // continue loop if store failed
- $acquire_success,
- "b 4f",
- "3:",
- // compare failed, set r to 1
- "mov {r}, #1",
- clrex!(),
- $acquire_failure,
- "4:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- r = out(reg) r,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- tmp = out(reg) _,
- )
- };
- }
- macro_rules! cmpxchg_release {
- ($acquire_failure:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- // (atomic) CAS (LL/SC loop)
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- dmb!(), // release
- "2:",
- concat!("strex", $asm_suffix, " {r}, {new}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 4f", // jump if store succeed
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "beq 2b", // continue loop if compare succeed
- "3:",
- // compare failed, set r to 1
- "mov {r}, #1",
- clrex!(),
- $acquire_failure,
- "4:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- r = out(reg) r,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- tmp = out(reg) _,
- )
- };
- }
- macro_rules! cmpxchg_acqrel {
- ($acquire_failure:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- // (atomic) CAS (LL/SC loop)
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- dmb!(), // release
- "2:",
- concat!("strex", $asm_suffix, " {r}, {new}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 4f", // jump if store succeed
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "beq 2b", // continue loop if compare succeed
- "3:",
- // compare failed, set r to 1
- "mov {r}, #1",
- clrex!(),
- $acquire_failure,
- "b 5f",
- "4:", // store succeed
- dmb!(), // acquire_success
- "5:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- r = out(reg) r,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- tmp = out(reg) _,
- )
- };
- }
- match (success, failure) {
- (Relaxed, Relaxed) => cmpxchg_store_relaxed!(asm_no_dmb, "", ""),
- (Relaxed, Acquire | SeqCst) => {
- cmpxchg_store_relaxed!(asm_use_dmb, "", dmb!());
- }
- (Acquire, Relaxed) => cmpxchg_store_relaxed!(asm_use_dmb, dmb!(), ""),
- (Acquire, Acquire | SeqCst) => {
- cmpxchg_store_relaxed!(asm_use_dmb, dmb!(), dmb!());
- }
- (Release, Relaxed) => cmpxchg_release!(""),
- (Release, Acquire | SeqCst) => cmpxchg_release!(dmb!()),
- // AcqRel and SeqCst compare_exchange are equivalent.
- (AcqRel | SeqCst, Relaxed) => cmpxchg_acqrel!(""),
- (AcqRel | SeqCst, _) => cmpxchg_acqrel!(dmb!()),
- _ => unreachable!(),
- }
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- #[inline]
- unsafe fn atomic_compare_exchange_weak(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- use core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst};
- let mut r: i32;
- macro_rules! cmpxchg_weak {
- ($asm:ident, $acquire:expr, $release:expr) => {
- $asm!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- $release,
- concat!("strex", $asm_suffix, " {r}, {new}, [{dst}]"),
- "b 4f",
- "3:",
- // compare failed, set r to 1
- "mov {r}, #1",
- clrex!(),
- "4:",
- $acquire,
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- r = out(reg) r,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- tmp = out(reg) _,
- )
- };
- }
- macro_rules! cmpxchg_weak_fail_load_relaxed {
- ($release:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- $release,
- concat!("strex", $asm_suffix, " {r}, {new}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 4f", // jump if store succeed
- "b 5f", // jump (store failed)
- "3:",
- // compare failed, set r to 1
- "mov {r}, #1",
- clrex!(),
- "b 5f",
- "4:", // store succeed
- dmb!(), // acquire_success
- "5:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- r = out(reg) r,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- tmp = out(reg) _,
- )
- };
- }
- macro_rules! cmpxchg_weak_success_load_relaxed {
- ($release:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- concat!("ldrex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- $release,
- concat!("strex", $asm_suffix, " {r}, {new}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 5f", // jump if store succeed
- "b 4f", // jump (store failed)
- "3:",
- // compare failed, set r to 1
- "mov {r}, #1",
- clrex!(),
- "4:", // compare or store failed
- dmb!(), // acquire_failure
- "5:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- r = out(reg) r,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- tmp = out(reg) _,
- )
- };
- }
- match (success, failure) {
- (Relaxed, Relaxed) => cmpxchg_weak!(asm_no_dmb, "", ""),
- (Relaxed, Acquire | SeqCst) => cmpxchg_weak_success_load_relaxed!(""),
- (Acquire, Relaxed) => cmpxchg_weak_fail_load_relaxed!(""),
- (Acquire, Acquire | SeqCst) => cmpxchg_weak!(asm_use_dmb, dmb!(), ""),
- (Release, Relaxed) => cmpxchg_weak!(asm_use_dmb, "", dmb!()),
- (Release, Acquire | SeqCst) => cmpxchg_weak_success_load_relaxed!(dmb!()),
- // AcqRel and SeqCst compare_exchange_weak are equivalent.
- (AcqRel | SeqCst, Relaxed) => cmpxchg_weak_fail_load_relaxed!(dmb!()),
- (AcqRel | SeqCst, _) => cmpxchg_weak!(asm_use_dmb, dmb!(), dmb!()),
- _ => unreachable!(),
- }
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic!(i8, "b");
-atomic!(u8, "b");
-atomic!(i16, "h");
-atomic!(u16, "h");
-atomic!(i32, "");
-atomic!(u32, "");
-atomic!(isize, "");
-atomic!(usize, "");
-
-// Refs:
-// - https://developer.arm.com/documentation/ddi0406/cb/Application-Level-Architecture/Instruction-Details/Alphabetical-list-of-instructions/LDREXD
-// - https://developer.arm.com/documentation/ddi0406/cb/Application-Level-Architecture/Instruction-Details/Alphabetical-list-of-instructions/STREXD
-#[rustfmt::skip]
-macro_rules! atomic64 {
- ($ty:ident) => {
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($asm:ident, $acquire:expr) => {
- $asm!(
- options(nostack, preserves_flags),
- // (atomic) load from src to tmp pair
- "ldrexd r2, r3, [{src}]",
- clrex!(),
- $acquire, // acquire fence
- // store tmp pair to out
- "strd r2, r3, [{out}]",
- src = in(reg) src,
- out = in(reg) out_ptr,
- // tmp pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!(asm_no_dmb, ""),
- // Acquire and SeqCst loads are equivalent.
- Ordering::Acquire | Ordering::SeqCst => atomic_load!(asm_use_dmb, dmb!()),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store {
- ($asm:ident, $acquire:expr, $release:expr) => {
- $asm!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from val to val pair
- "ldrd r2, r3, [{val}]",
- // (atomic) store val pair to dst (LL/SC loop)
- $release, // release fence
- "2:",
- // load from dst to tmp pair
- "ldrexd r4, r5, [{dst}]",
- // try to store val pair to dst
- "strexd {r}, r2, r3, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b",
- $acquire, // acquire fence
- dst = inout(reg) dst => _,
- val = in(reg) val,
- r = lateout(reg) _,
- // val pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // tmp pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!(asm_no_dmb, "", ""),
- Ordering::Release => atomic_store!(asm_use_dmb, "", dmb!()),
- Ordering::SeqCst => atomic_store!(asm_use_dmb, dmb!(), dmb!()),
- _ => unreachable!(),
- }
- }
- }
- }
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_swap {
- ($asm:ident, $acquire:expr, $release:expr) => {
- $asm!(
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- // load from val to val pair
- "ldrd r2, r3, [{val}]",
- // (atomic) swap (LL/SC loop)
- $release, // release fence
- "2:",
- // load from dst to out pair
- "ldrexd r4, r5, [{dst}]",
- // try to store val pair to dst
- "strexd {r}, r2, r3, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b",
- $acquire, // acquire fence
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- val = in(reg) val,
- out = inout(reg) out_ptr => _,
- r = lateout(reg) _,
- // val pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_swap!(asm_no_dmb, "", ""),
- Ordering::Acquire => atomic_swap!(asm_use_dmb, dmb!(), ""),
- Ordering::Release => atomic_swap!(asm_use_dmb, "", dmb!()),
- // AcqRel and SeqCst swaps are equivalent.
- Ordering::AcqRel | Ordering::SeqCst => atomic_swap!(asm_use_dmb, dmb!(), dmb!()),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- use core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst};
- let mut r: i32;
- macro_rules! cmpxchg_store_relaxed {
- ($asm:ident, $acquire_success:expr, $acquire_failure:expr) => {
- $asm!(
- // Do not use `preserves_flags` because CMP and ORRS modify the condition flags.
- options(nostack),
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- // (atomic) CAS (LL/SC loop)
- "2:",
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- "strexd {r}, r8, r9, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b", // continue loop if store failed
- $acquire_success,
- "b 4f",
- "3:",
- // compare failed, set r to 1 and clear exclusive
- "mov {r}, #1",
- clrex!(),
- $acquire_failure,
- "4:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- )
- };
- }
- macro_rules! cmpxchg_release {
- ($acquire_failure:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP and ORRS modify the condition flags.
- options(nostack),
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- // (atomic) CAS (LL/SC loop)
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- dmb!(), // release
- "2:",
- "strexd {r}, r8, r9, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 4f", // jump if store succeed
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "beq 2b", // continue loop if compare succeed
- "3:",
- // compare failed, set r to 1 and clear exclusive
- "mov {r}, #1",
- clrex!(),
- $acquire_failure,
- "4:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- )
- };
- }
- macro_rules! cmpxchg_acqrel {
- ($acquire_failure:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP and ORRS modify the condition flags.
- options(nostack),
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- // (atomic) CAS (LL/SC loop)
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- dmb!(), // release
- "2:",
- "strexd {r}, r8, r9, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 4f", // jump if store succeed
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "beq 2b", // continue loop if compare succeed
- "3:",
- // compare failed, set r to 1 and clear exclusive
- "mov {r}, #1",
- clrex!(),
- $acquire_failure,
- "b 5f",
- "4:", // store succeed
- dmb!(), // acquire_success
- "5:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- )
- };
- }
- match (success, failure) {
- (Relaxed, Relaxed) => cmpxchg_store_relaxed!(asm_no_dmb, "", ""),
- (Relaxed, Acquire | SeqCst) => cmpxchg_store_relaxed!(asm_use_dmb, "", dmb!()),
- (Acquire, Relaxed) => cmpxchg_store_relaxed!(asm_use_dmb, dmb!(), ""),
- (Acquire, Acquire | SeqCst) => cmpxchg_store_relaxed!(asm_use_dmb, dmb!(), dmb!()),
- (Release, Relaxed) => cmpxchg_release!(""),
- (Release, Acquire | SeqCst) => cmpxchg_release!(dmb!()),
- // AcqRel and SeqCst compare_exchange are equivalent.
- (AcqRel | SeqCst, Relaxed) => cmpxchg_acqrel!(""),
- (AcqRel | SeqCst, _) => cmpxchg_acqrel!(dmb!()),
- _ => unreachable!(),
- }
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- #[inline]
- unsafe fn atomic_compare_exchange_weak(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- use core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst};
- let mut r: i32;
- macro_rules! cmpxchg_weak {
- ($asm:ident, $acquire:expr, $release:expr) => {
- $asm!(
- // Do not use `preserves_flags` because ORRS modifies the condition flags.
- options(nostack),
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- $release,
- "strexd {r}, r8, r9, [{dst}]",
- "b 4f",
- "3:",
- // compare failed, set r to 1 and clear exclusive
- "mov {r}, #1",
- clrex!(),
- "4:",
- $acquire,
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- )
- };
- }
- macro_rules! cmpxchg_weak_fail_load_relaxed {
- ($release:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP and ORRS modify the condition flags.
- options(nostack),
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- $release,
- "strexd {r}, r8, r9, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 4f", // jump if store succeed
- "b 5f", // jump (store failed)
- "3:",
- // compare failed, set r to 1 and clear exclusive
- "mov {r}, #1",
- clrex!(),
- "b 5f",
- "4:", // store succeed
- dmb!(), // acquire_success
- "5:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- )
- };
- }
- macro_rules! cmpxchg_weak_success_load_relaxed {
- ($release:expr) => {
- asm_use_dmb!(
- // Do not use `preserves_flags` because CMP and ORRS modify the condition flags.
- options(nostack),
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- "ldrexd r4, r5, [{dst}]",
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- $release,
- "strexd {r}, r8, r9, [{dst}]",
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "beq 5f", // jump if store succeed
- "b 4f", // jump (store failed)
- "3:",
- // compare failed, set r to 1 and clear exclusive
- "mov {r}, #1",
- clrex!(),
- "4:", // compare or store failed
- dmb!(), // acquire_failure
- "5:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- )
- };
- }
- match (success, failure) {
- (Relaxed, Relaxed) => cmpxchg_weak!(asm_no_dmb, "", ""),
- (Relaxed, Acquire | SeqCst) => cmpxchg_weak_success_load_relaxed!(""),
- (Acquire, Relaxed) => cmpxchg_weak_fail_load_relaxed!(""),
- (Acquire, Acquire | SeqCst) => cmpxchg_weak!(asm_use_dmb, dmb!(), ""),
- (Release, Relaxed) => cmpxchg_weak!(asm_use_dmb, "", dmb!()),
- (Release, Acquire | SeqCst) => cmpxchg_weak_success_load_relaxed!(dmb!()),
- // AcqRel and SeqCst compare_exchange_weak are equivalent.
- (AcqRel | SeqCst, Relaxed) => cmpxchg_weak_fail_load_relaxed!(dmb!()),
- (AcqRel | SeqCst, _) => cmpxchg_weak!(asm_use_dmb, dmb!(), dmb!()),
- _ => unreachable!(),
- }
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic64!(i64);
-atomic64!(u64);
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-))]
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-))]
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-)))]
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(any(
- any(target_feature = "v7", atomic_maybe_uninit_target_feature = "v7"),
- not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")),
-)))]
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
diff --git a/src/arch_legacy/arm_linux.rs b/src/arch_legacy/arm_linux.rs
deleted file mode 100644
index a4d90734..00000000
--- a/src/arch_legacy/arm_linux.rs
+++ /dev/null
@@ -1,663 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-Pre-v6 Arm Linux/Android
-
-See arch/arm_linux.rs for references and notes.
-
-Generated asm:
-- armv5te https://godbolt.org/z/r61s7cnG8
-- armv4t https://godbolt.org/z/xrxfKx1rc
-*/
-
-use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering};
-
-use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
-
-// https://github.com/torvalds/linux/blob/v6.16/Documentation/arch/arm/kernel_user_helpers.rst
-const KUSER_HELPER_VERSION: usize = 0xFFFF0FFC;
-// __kuser_helper_version >= 2 (kernel version 2.6.12+)
-const KUSER_CMPXCHG: usize = 0xFFFF0FC0;
-// __kuser_helper_version >= 3 (kernel version 2.6.15+)
-const KUSER_MEMORY_BARRIER: usize = 0xFFFF0FA0;
-// __kuser_helper_version >= 5 (kernel version 3.1+)
-const KUSER_CMPXCHG64: usize = 0xFFFF0F60;
-
-#[inline]
-fn kuser_helper_version() -> i32 {
- // SAFETY: core assumes that at least __kuser_memory_barrier (__kuser_helper_version >= 3) is
- // available on this platform. __kuser_helper_version is always available on such a platform.
- unsafe { crate::utils::ptr::with_exposed_provenance::(KUSER_HELPER_VERSION).read() }
-}
-
-#[cfg(any(target_feature = "v5te", atomic_maybe_uninit_target_feature = "v5te"))]
-macro_rules! blx {
- ($addr:tt) => {
- concat!("blx ", $addr)
- };
-}
-#[cfg(not(any(target_feature = "v5te", atomic_maybe_uninit_target_feature = "v5te")))]
-macro_rules! blx {
- ($addr:tt) => {
- concat!("mov lr, pc", "\n", "bx ", $addr)
- };
-}
-
-macro_rules! atomic_load_store {
- ($ty:ident, $asm_suffix:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- match order {
- Ordering::Relaxed => {
- asm!(
- // (atomic) load from src to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{src}]"),
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- src = in(reg) src,
- out = inout(reg) out_ptr => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- );
- }
- // Acquire and SeqCst loads are equivalent.
- Ordering::Acquire | Ordering::SeqCst => {
- debug_assert!(kuser_helper_version() >= 3);
- asm!(
- // (atomic) load from src to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{src}]"),
- blx!("{kuser_memory_barrier}"), // acquire fence
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- src = in(reg) src,
- out = inout(reg) out_ptr => _,
- tmp = lateout(reg) _,
- kuser_memory_barrier = inout(reg) KUSER_MEMORY_BARRIER => _,
- out("lr") _,
- options(nostack, preserves_flags),
- );
- }
- _ => unreachable!(),
- }
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store_release {
- ($acquire:expr) => {{
- debug_assert!(kuser_helper_version() >= 3);
- asm!(
- // load from val to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{val}]"),
- // (atomic) store tmp to dst
- blx!("{kuser_memory_barrier}"), // release fence
- concat!("str", $asm_suffix, " {tmp}, [{dst}]"),
- $acquire, // acquire fence
- dst = inout(reg) dst => _,
- val = in(reg) val,
- tmp = lateout(reg) _,
- kuser_memory_barrier = inout(reg) KUSER_MEMORY_BARRIER => _,
- out("lr") _,
- options(nostack, preserves_flags),
- )
- }};
- }
- match order {
- Ordering::Relaxed => {
- asm!(
- // load from val to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{val}]"),
- // (atomic) store tmp to dst
- concat!("str", $asm_suffix, " {tmp}, [{dst}]"),
- dst = inout(reg) dst => _,
- val = in(reg) val,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- );
- }
- Ordering::Release => atomic_store_release!(""),
- Ordering::SeqCst => atomic_store_release!(blx!("{kuser_memory_barrier}")),
- _ => unreachable!(),
- }
- }
- }
- }
- };
-}
-
-macro_rules! atomic {
- ($ty:ident) => {
- atomic_load_store!($ty, "");
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- debug_assert!(kuser_helper_version() >= 2);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- asm!(
- "ldr r1, [r1]", // new_val
- "2:",
- "ldr r0, [r2]", // old_val
- "mov {out_tmp}, r0",
- blx!("{kuser_cmpxchg}"),
- "cmp r0, #0",
- "bne 2b",
- "str {out_tmp}, [{out}]",
- out = in(reg) out_ptr,
- out_tmp = out(reg) _,
- kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
- out("r0") _,
- inout("r1") val => _,
- in("r2") dst, // ptr
- out("r3") _,
- out("ip") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg modify the condition flags.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- _failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- debug_assert!(kuser_helper_version() >= 2);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- asm!(
- "ldr {old}, [{old}]",
- "ldr {new}, [{new}]",
- "2:",
- "ldr r0, [r2]", // old_val
- "mov {out_tmp}, r0",
- "cmp r0, {old}",
- "bne 3f",
- "mov r1, {new}", // new_val
- blx!("{kuser_cmpxchg}"),
- "cmp r0, #0",
- "bne 2b",
- "b 4f",
- "3:",
- // write back to synchronize
- "mov r1, r0", // new_val
- blx!("{kuser_cmpxchg}"),
- "cmp r0, #0",
- "bne 2b",
- "mov r0, #1",
- "4:",
- "str {out_tmp}, [{out}]",
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- out_tmp = out(reg) _,
- kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
- out("r0") r,
- out("r1") _,
- in("r2") dst, // ptr
- out("r3") _,
- out("ip") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg modify the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-macro_rules! atomic_sub_word {
- ($ty:ident, $asm_suffix:tt) => {
- atomic_load_store!($ty, $asm_suffix);
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- debug_assert!(kuser_helper_version() >= 2);
- let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- // Implement sub-word atomic operations using word-sized LL/SC loop.
- // See also create_sub_word_mask_values.
- asm!(
- concat!("ldr", $asm_suffix, " {val}, [{val}]"),
- "lsl {val}, {val}, {shift}",
- "and {val}, {val}, {mask}",
- "2:",
- "ldr r0, [r2]", // old_val
- "mov {out_tmp}, r0",
- "and r1, r0, {inv_mask}",
- "orr r1, r1, {val}", // new_val
- blx!("{kuser_cmpxchg}"),
- "cmp r0, #0",
- "bne 2b",
- "lsr {out_tmp}, {out_tmp}, {shift}",
- concat!("str", $asm_suffix, " {out_tmp}, [{out}]"),
- val = inout(reg) val => _,
- out = in(reg) out_ptr,
- shift = in(reg) shift,
- mask = in(reg) mask,
- inv_mask = in(reg) !mask,
- out_tmp = out(reg) _,
- kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
- out("r0") _,
- out("r1") _,
- in("r2") dst, // ptr
- out("r3") _,
- out("ip") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg modify the condition flags.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- _failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- debug_assert!(kuser_helper_version() >= 2);
- let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- // Implement sub-word atomic operations using word-sized LL/SC loop.
- // See also create_sub_word_mask_values.
- asm!(
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- "lsl {old}, {old}, {shift}",
- "lsl {new}, {new}, {shift}",
- "and {old}, {old}, {mask}",
- "and {new}, {new}, {mask}",
- "2:",
- "ldr r0, [r2]", // old_val
- "and {out_tmp}, r0, {mask}",
- "cmp {out_tmp}, {old}",
- "bne 3f",
- "mvn r1, {mask}",
- "and r1, r0, r1",
- "orr r1, r1, {new}", // new_val
- blx!("{kuser_cmpxchg}"),
- "cmp r0, #0",
- "bne 2b",
- "b 4f",
- "3:",
- // write back to synchronize
- "mov r1, r0", // new_val
- blx!("{kuser_cmpxchg}"),
- "cmp r0, #0",
- "bne 2b",
- "mov r0, #1",
- "4:",
- "lsr {out_tmp}, {out_tmp}, {shift}",
- concat!("str", $asm_suffix, " {out_tmp}, [{out}]"),
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- shift = in(reg) shift,
- mask = in(reg) mask,
- // We cannot create inv_mask here because there are no available registers
- // inv_mask = in(reg) !mask,
- out_tmp = out(reg) _,
- kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
- out("r0") r,
- out("r1") _,
- in("r2") dst, // ptr
- out("r3") _,
- out("ip") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg modify the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic_sub_word!(i8, "b");
-atomic_sub_word!(u8, "b");
-atomic_sub_word!(i16, "h");
-atomic_sub_word!(u16, "h");
-atomic!(i32);
-atomic!(u32);
-atomic!(isize);
-atomic!(usize);
-
-macro_rules! atomic64 {
- ($ty:ident) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- assert_has_kuser_cmpxchg64();
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- asm!(
- "2:",
- "ldr r0, [r2]",
- "ldr r3, [r2, #4]",
- "str r0, [r1]",
- "str r3, [r1, #4]",
- "mov r0, r1", // old_val
- blx!("{kuser_cmpxchg64}"),
- "cmp r0, #0",
- "bne 2b",
- kuser_cmpxchg64 = in(reg) KUSER_CMPXCHG64,
- out("r0") _,
- in("r1") out_ptr, // new_val
- in("r2") src, // ptr
- out("r3") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg64 modify the condition flags.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- assert_has_kuser_cmpxchg64();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut out_tmp = MaybeUninit::::uninit();
- asm!(
- "2:",
- "ldr r0, [r2]",
- "ldr r3, [r2, #4]",
- "str r0, [{out_tmp}]",
- "str r3, [{out_tmp}, #4]",
- "mov r0, {out_tmp}", // old_val
- blx!("{kuser_cmpxchg64}"),
- "cmp r0, #0",
- "bne 2b",
- out_tmp = in(reg) out_tmp.as_mut_ptr(),
- kuser_cmpxchg64 = in(reg) KUSER_CMPXCHG64,
- out("r0") _,
- in("r1") val, // new_val
- in("r2") dst, // ptr
- out("r3") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg64 modify the condition flags.
- options(nostack),
- );
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- assert_has_kuser_cmpxchg64();
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- asm!(
- "2:",
- "ldr r0, [r2]",
- "ldr r3, [r2, #4]",
- "str r0, [{out_tmp}]",
- "str r3, [{out_tmp}, #4]",
- "mov r0, {out_tmp}", // old_val
- blx!("{kuser_cmpxchg64}"),
- "cmp r0, #0",
- "bne 2b",
- out_tmp = in(reg) out_ptr,
- kuser_cmpxchg64 = in(reg) KUSER_CMPXCHG64,
- out("r0") _,
- in("r1") val, // new_val
- in("r2") dst, // ptr
- out("r3") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP and __kuser_cmpxchg64 modify the condition flags.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- _failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- assert_has_kuser_cmpxchg64();
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- asm!(
- "ldr {old_lo}, [{old_hi}]",
- "ldr {old_hi}, [{old_hi}, #4]",
- "2:",
- "ldr r0, [r2]",
- "ldr r3, [r2, #4]",
- "str r0, [{out_tmp}]",
- "str r3, [{out_tmp}, #4]",
- "eor r0, r0, {old_lo}",
- "eor r3, r3, {old_hi}",
- "orrs r0, r0, r3",
- "bne 3f",
- "mov r0, {out_tmp}", // old_val
- "mov r1, {new}", // new_val
- blx!("{kuser_cmpxchg64}"),
- "cmp r0, #0",
- "bne 2b",
- "b 4f",
- "3:",
- // write back to ensure atomicity
- "mov r0, {out_tmp}", // old_val
- "mov r1, {out_tmp}", // new_val
- blx!("{kuser_cmpxchg64}"),
- "cmp r0, #0",
- "bne 2b",
- "mov r0, #1",
- "4:",
- new = in(reg) new,
- out_tmp = in(reg) out_ptr,
- old_lo = out(reg) _,
- old_hi = inout(reg) old => _,
- kuser_cmpxchg64 = in(reg) KUSER_CMPXCHG64,
- out("r0") r,
- out("r1") _,
- in("r2") dst, // ptr
- out("r3") _,
- out("lr") _,
- // Do not use `preserves_flags` because CMP, ORRS, and __kuser_cmpxchg64 modify the condition flags.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic64!(i64);
-atomic64!(u64);
-
-// TODO: Since Rust 1.64, the Linux kernel requirement for Rust when using std is 3.2+, so it
-// should be possible to convert this to debug_assert if the std feature is enabled on Rust 1.64+.
-// https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements.html
-#[inline]
-fn assert_has_kuser_cmpxchg64() {
- if kuser_helper_version() < 5 {
- #[cold]
- fn p() -> ! {
- panic!("64-bit atomics on pre-v6 Arm requires Linux kernel version 3.1+")
- }
- p()
- }
-}
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-// TODO: set has_atomic_64 to true
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
-
-#[cfg(test)]
-mod tests {
- #[test]
- fn kuser_helper_version() {
- let version = super::kuser_helper_version();
- assert!(version >= 5, "{version:?}");
- }
-
- // TODO: set has_atomic_64 to true
- test_atomic!(i64);
- test_atomic!(u64);
- stress_test!(u64);
-}
diff --git a/src/arch_legacy/armv8.rs b/src/arch_legacy/armv8.rs
deleted file mode 100644
index a615b503..00000000
--- a/src/arch_legacy/armv8.rs
+++ /dev/null
@@ -1,634 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-Armv8 AArch32
-
-See arch/armv8.rs for references and notes.
-
-Generated asm:
-- armv8-a https://godbolt.org/z/Mx8z81463
-- armv8-m baseline https://godbolt.org/z/P51ezojjW
-- armv8-m mainline https://godbolt.org/z/WdajnbYTr
-*/
-
-use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering};
-
-use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
-
-macro_rules! atomic_rmw {
- ($op:ident, $order:ident) => {
- match $order {
- Ordering::Relaxed => $op!("r", "r"),
- Ordering::Acquire => $op!("a", "r"),
- Ordering::Release => $op!("r", "l"),
- // AcqRel and SeqCst RMWs are equivalent.
- Ordering::AcqRel | Ordering::SeqCst => $op!("a", "l"),
- _ => unreachable!(),
- }
- };
-}
-
-// Adds S suffix if needed. We prefer instruction without S suffix,
-// but Armv8-M Baseline doesn't support thumb2 instructions.
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-macro_rules! s {
- ($op:tt, $operand:tt) => {
- concat!($op, " ", $operand)
- };
-}
-#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
-macro_rules! s {
- ($op:tt, $operand:tt) => {
- concat!($op, "s ", $operand)
- };
-}
-
-macro_rules! atomic {
- ($ty:ident, $asm_suffix:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($acquire:tt) => {
- asm!(
- // (atomic) load from src to tmp
- concat!("ld", $acquire, $asm_suffix, " {tmp}, [{src}]"),
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- src = in(reg) src,
- out = inout(reg) out_ptr => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!("r"),
- // Acquire and SeqCst loads are equivalent.
- Ordering::Acquire | Ordering::SeqCst => atomic_load!("a"),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store {
- ($release:tt) => {
- asm!(
- // load from val to tmp
- concat!("ldr", $asm_suffix, " {tmp}, [{val}]"),
- // (atomic) store tmp to dst
- concat!("st", $release, $asm_suffix, " {tmp}, [{dst}]"),
- dst = inout(reg) dst => _,
- val = in(reg) val,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!("r"),
- // Release and SeqCst stores are equivalent.
- Ordering::Release | Ordering::SeqCst => atomic_store!("l"),
- _ => unreachable!(),
- }
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! swap {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from val (ptr) to val (val)
- concat!("ldr", $asm_suffix, " {val}, [{val}]"),
- // (atomic) swap (LL/SC loop)
- "2:",
- // load from dst to tmp
- concat!("ld", $acquire, "ex", $asm_suffix, " {tmp}, [{dst}]"),
- // try to store val to dst
- concat!("st", $release, "ex", $asm_suffix, " {r}, {val}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- val = inout(reg) val => _,
- out = in(reg) out_ptr,
- r = out(reg) _,
- tmp = out(reg) _,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(swap, order);
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- // (atomic) CAS (LL/SC loop)
- "2:",
- // load from dst to tmp
- concat!("ld", $acquire, "ex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f", // jump if compare failed
- // try to store val to dst
- concat!("st", $release, "ex", $asm_suffix, " {r}, {new}, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b", // continue loop if store failed
- "b 4f",
- "3:",
- // compare failed, mark r as failed and clear exclusive
- "clrex",
- s!("mov", "{r}, #1"),
- "4:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- r = out(reg) r,
- tmp = out(reg) _,
- // Do not use `preserves_flags` because CMP and s! modify the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(cmpxchg, order);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- #[inline]
- unsafe fn atomic_compare_exchange_weak(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- macro_rules! cmpxchg_weak {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from old/new (ptr) to old/new (val)
- concat!("ldr", $asm_suffix, " {old}, [{old}]"),
- concat!("ldr", $asm_suffix, " {new}, [{new}]"),
- // load from dst to tmp
- concat!("ld", $acquire, "ex", $asm_suffix, " {tmp}, [{dst}]"),
- "cmp {tmp}, {old}",
- "bne 3f",
- // try to store new to dst
- concat!("st", $release, "ex", $asm_suffix, " {r}, {new}, [{dst}]"),
- "b 4f",
- "3:",
- // compare failed, mark r as failed and clear exclusive
- "clrex",
- s!("mov", "{r}, #1"),
- "4:",
- // store tmp to out
- concat!("str", $asm_suffix, " {tmp}, [{out}]"),
- dst = in(reg) dst,
- old = inout(reg) old => _,
- new = inout(reg) new => _,
- out = in(reg) out_ptr,
- r = out(reg) r,
- tmp = out(reg) _,
- // Do not use `preserves_flags` because CMP and s! modify the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(cmpxchg_weak, order);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic!(i8, "b");
-atomic!(u8, "b");
-atomic!(i16, "h");
-atomic!(u16, "h");
-atomic!(i32, "");
-atomic!(u32, "");
-atomic!(isize, "");
-atomic!(usize, "");
-
-#[rustfmt::skip]
-macro_rules! atomic64 {
- ($ty:ident) => {
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($acquire:tt) => {
- asm!(
- // (atomic) load from src to tmp pair
- concat!("ld", $acquire, "exd r2, r3, [{src}]"),
- "clrex",
- // store tmp pair to out
- "strd r2, r3, [{out}]",
- src = in(reg) src,
- out = in(reg) out_ptr,
- // tmp pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!("r"),
- // Acquire and SeqCst loads are equivalent.
- Ordering::Acquire | Ordering::SeqCst => atomic_load!("a"),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! store {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from val to val pair
- "ldrd r2, r3, [{val}]",
- // (atomic) store val pair to dst (LL/SC loop)
- "2:",
- // load from dst to tmp pair
- concat!("ld", $acquire, "exd r4, r5, [{dst}]"),
- // try to store val pair to dst
- concat!("st", $release, "exd {r}, r2, r3, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b",
- dst = inout(reg) dst => _,
- val = in(reg) val,
- r = lateout(reg) _,
- // val pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // tmp pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(store, order);
- }
- }
- }
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! swap {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from val to val pair
- "ldrd r2, r3, [{val}]",
- // (atomic) swap (LL/SC loop)
- "2:",
- // load from dst to out pair
- concat!("ld", $acquire, "exd r4, r5, [{dst}]"),
- // try to store val pair to dst
- concat!("st", $release, "exd {r}, r2, r3, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- val = in(reg) val,
- out = inout(reg) out_ptr => _,
- r = lateout(reg) _,
- // val pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // Do not use `preserves_flags` because CMP modifies the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(swap, order);
- }
- out
- }
- }
- #[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt) => {
- asm!(
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- // (atomic) CAS (LL/SC loop)
- "2:",
- concat!("ld", $acquire, "exd r4, r5, [{dst}]"),
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- concat!("st", $release, "exd {r}, r8, r9, [{dst}]"),
- // 0 if the store was successful, 1 if no store was performed
- "cmp {r}, #0",
- "bne 2b", // continue loop if store failed
- "b 4f",
- "3:",
- // compare failed, mark r as failed and clear exclusive
- "clrex",
- s!("mov", "{r}, #1"),
- "4:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- // Do not use `preserves_flags` because CMP, ORRS, and s! modify the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(cmpxchg, order);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- #[inline]
- unsafe fn atomic_compare_exchange_weak(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: i32;
- macro_rules! cmpxchg_weak {
- ($acquire:tt, $release:tt) => {
- asm!(
- "ldrd r2, r3, [{old}]",
- "ldrd r8, r9, [{new}]",
- concat!("ld", $acquire, "exd r4, r5, [{dst}]"),
- "eor {tmp}, r5, r3",
- "eor {r}, r4, r2",
- "orrs {r}, {r}, {tmp}",
- "bne 3f", // jump if compare failed
- concat!("st", $release, "exd {r}, r8, r9, [{dst}]"),
- "b 4f",
- "3:",
- // compare failed, mark r as failed and clear exclusive
- "clrex",
- s!("mov", "{r}, #1"),
- "4:",
- // store out pair to out
- "strd r4, r5, [{out}]",
- dst = inout(reg) dst => _,
- r = lateout(reg) r,
- old = in(reg) old,
- new = in(reg) new,
- out = inout(reg) out_ptr => _,
- tmp = out(reg) _,
- // old pair - must be even-numbered and not R14
- out("r2") _,
- out("r3") _,
- // out pair - must be even-numbered and not R14
- out("r4") _,
- out("r5") _,
- // new pair - must be even-numbered and not R14
- out("r8") _,
- out("r9") _,
- // Do not use `preserves_flags` because ORRS and s! modify the condition flags.
- options(nostack),
- )
- };
- }
- atomic_rmw!(cmpxchg_weak, order);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- // 0 if the store was successful, 1 if no store was performed
- (out, r == 0)
- }
- }
- }
- };
-}
-
-atomic64!(i64);
-atomic64!(u64);
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[cfg(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass"))]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(any(target_feature = "mclass", atomic_maybe_uninit_target_feature = "mclass")))]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
diff --git a/src/arch_legacy/loongarch.rs b/src/arch_legacy/loongarch.rs
deleted file mode 100644
index 9f4fe1cf..00000000
--- a/src/arch_legacy/loongarch.rs
+++ /dev/null
@@ -1,443 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-LoongArch
-
-See arch/loongarch.rs for references and notes.
-
-Generated asm:
-- loongarch64 https://godbolt.org/z/vTxfajT14
-*/
-
-use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering};
-
-use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
-
-#[rustfmt::skip]
-macro_rules! atomic_load {
- ($ty:ident, $asm_suffix:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($acquire:tt) => {
- asm!(
- // (atomic) load from src to tmp
- concat!("ld.", $asm_suffix, " {tmp}, {src}, 0"),
- $acquire,
- // store tmp to out
- concat!("st.", $asm_suffix, " {tmp}, {out}, 0"),
- src = in(reg) ptr_reg!(src),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!(""),
- Ordering::Acquire => atomic_load!("dbar 20"),
- Ordering::SeqCst => atomic_load!("dbar 16"),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- };
-}
-
-macro_rules! atomic {
- ($ty:ident, $asm_suffix:tt) => {
- atomic_load!($ty, $asm_suffix);
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- match order {
- Ordering::Relaxed => {
- asm!(
- // load from val to tmp
- concat!("ld.", $asm_suffix, " {tmp}, {val}, 0"),
- // (atomic) store tmp to dst
- concat!("st.", $asm_suffix, " {tmp}, {dst}, 0"),
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- );
- }
- Ordering::Release | Ordering::SeqCst => {
- asm!(
- // load from val to tmp
- concat!("ld.", $asm_suffix, " {tmp}, {val}, 0"),
- // (atomic) store tmp to dst
- concat!("amswap_db.", $asm_suffix, " $zero, {tmp}, {dst}"),
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- }
- _ => unreachable!(),
- }
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- // AMO is always SeqCst.
- asm!(
- // load from val (ptr) to val (val)
- concat!("ld.", $asm_suffix, " {val}, {val}, 0"),
- // (atomic) swap (AMO)
- // - load value from dst and store it to tmp
- // - store value of val to dst
- concat!("amswap_db.", $asm_suffix, " {tmp}, {val}, {dst}"),
- // store tmp to out
- concat!("st.", $asm_suffix, " {tmp}, {out}, 0"),
- dst = in(reg) ptr_reg!(dst),
- val = inout(reg) ptr_reg!(val) => _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = out(reg) _,
- options(nostack, preserves_flags),
- )
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: crate::utils::RegSize;
- macro_rules! cmpxchg {
- ($failure_fence:tt) => {
- asm!(
- // load from old/new (ptr) to old/new (val)
- concat!("ld.", $asm_suffix, " {old}, {old}, 0"),
- concat!("ld.", $asm_suffix, " {new}, {new}, 0"),
- // (atomic) CAS (LL/SC loop)
- "2:",
- concat!("ll.", $asm_suffix, " {tmp}, {dst}, 0"),
- "bne {tmp}, {old}, 3f", // compare and jump if compare failed
- "move {r}, {new}",
- concat!("sc.", $asm_suffix, " {r}, {dst}, 0"),
- "beqz {r}, 2b", // continue loop if store failed
- "b 4f",
- "3:",
- $failure_fence,
- "4:",
- // store tmp to out
- concat!("st.", $asm_suffix, " {tmp}, {out}, 0"),
- "xor {r}, {tmp}, {old}",
- "sltui {r}, {r}, 1",
- dst = in(reg) ptr_reg!(dst),
- old = inout(reg) ptr_reg!(old) => _,
- new = inout(reg) ptr_reg!(new) => _,
- out = in(reg) ptr_reg!(out_ptr),
- tmp = out(reg) _,
- r = out(reg) r,
- options(nostack, preserves_flags),
- )
- };
- }
- // LL/SC is always SeqCst, and fence is needed for branch that doesn't call sc.
- match failure {
- Ordering::Relaxed => cmpxchg!("dbar 1792"),
- Ordering::Acquire => cmpxchg!("dbar 20"),
- // TODO: LLVM uses dbar 20 (Acquire) here, but should it not be dbar 16 (SeqCst)?
- Ordering::SeqCst => cmpxchg!("dbar 16"),
- _ => unreachable!(),
- }
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-macro_rules! atomic_sub_word {
- ($ty:ident, $asm_suffix:tt) => {
- atomic_load!($ty, $asm_suffix);
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from val to tmp
- concat!("ld.", $asm_suffix, " {tmp}, {val}, 0"),
- // (atomic) store tmp to dst
- $release,
- concat!("st.", $asm_suffix, " {tmp}, {dst}, 0"),
- $acquire,
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!("", ""),
- Ordering::Release => atomic_store!("", "dbar 18"),
- Ordering::SeqCst => atomic_store!("dbar 16", "dbar 16"),
- _ => unreachable!(),
- }
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- // Implement sub-word atomic operations using word-sized LL/SC loop.
- // Based on assemblies generated by rustc/LLVM.
- // See also create_sub_word_mask_values.
- asm!(
- "sll.w {mask}, {mask}, {shift}",
- "addi.w {mask}, {mask}, 0",
- concat!("ld.", $asm_suffix, "u {val}, {val}, 0"),
- "sll.w {val}, {val}, {shift}",
- "addi.w {val}, {val}, 0",
- // (atomic) swap (LL/SC loop)
- "2:",
- "ll.w {tmp1}, {dst}, 0",
- "addi.w {tmp2}, {val}, 0",
- "xor {tmp2}, {tmp1}, {tmp2}",
- "and {tmp2}, {tmp2}, {mask}",
- "xor {tmp2}, {tmp1}, {tmp2}",
- "sc.w {tmp2}, {dst}, 0",
- "beqz {tmp2}, 2b",
- "srl.w {tmp1}, {tmp1}, {shift}",
- concat!("st.", $asm_suffix, " {tmp1}, {out}, 0"),
- dst = in(reg) ptr_reg!(dst),
- val = inout(reg) ptr_reg!(val) => _,
- out = in(reg) ptr_reg!(out_ptr),
- shift = in(reg) shift,
- mask = inout(reg) mask => _,
- tmp1 = out(reg) _,
- tmp2 = out(reg) _,
- options(nostack, preserves_flags),
- );
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: crate::utils::RegSize;
- // Implement sub-word atomic operations using word-sized LL/SC loop.
- // Based on assemblies generated by rustc/LLVM.
- // See also create_sub_word_mask_values.
- macro_rules! cmpxchg {
- ($failure_fence:tt) => {
- asm!(
- concat!("ld.", $asm_suffix, "u {new}, {new}, 0"),
- concat!("ld.", $asm_suffix, "u {old}, {old}, 0"),
- "sll.w {new}, {new}, {shift}",
- "addi.w {new}, {new}, 0",
- "sll.w {old}, {old}, {shift}",
- "addi.w $a7, {old}, 0",
- "sll.w {mask}, {mask}, {shift}",
- "addi.w $a6, {mask}, 0",
- // (atomic) CAS (LL/SC loop)
- "2:",
- "ll.w $t0, {dst}, 0",
- "and $t1, $t0, $a6",
- "bne $t1, $a7, 3f",
- "andn $t1, $t0, $a6",
- "or $t1, $t1, {new}",
- "sc.w $t1, {dst}, 0",
- "beqz $t1, 2b",
- "b 4f",
- "3:",
- $failure_fence,
- "4:",
- "srl.w $a6, $t0, {shift}",
- concat!("st.", $asm_suffix, " $a6, {out}, 0"),
- "and {r}, $t0, {mask}",
- "addi.w {r}, {r}, 0",
- "xor {r}, {old}, {r}",
- "sltui {r}, {r}, 1",
- dst = in(reg) ptr_reg!(dst),
- old = inout(reg) ptr_reg!(old) => _,
- new = inout(reg) ptr_reg!(new) => _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- shift = in(reg) shift,
- mask = inout(reg) mask => _,
- r = lateout(reg) r,
- out("$a6") _,
- out("$a7") _,
- out("$t0") _,
- out("$t1") _,
- options(nostack, preserves_flags),
- )
- };
- }
- // LL/SC is always SeqCst, and fence is needed for branch that doesn't call sc.
- match failure {
- Ordering::Relaxed => cmpxchg!("dbar 1792"),
- Ordering::Acquire => cmpxchg!("dbar 20"),
- // TODO: LLVM uses dbar 20 (Acquire) here, but should it not be dbar 16 (SeqCst)?
- Ordering::SeqCst => cmpxchg!("dbar 16"),
- _ => unreachable!(),
- }
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-atomic_sub_word!(i8, "b");
-atomic_sub_word!(u8, "b");
-atomic_sub_word!(i16, "h");
-atomic_sub_word!(u16, "h");
-atomic!(i32, "w");
-atomic!(u32, "w");
-#[cfg(target_arch = "loongarch64")]
-atomic!(i64, "d");
-#[cfg(target_arch = "loongarch64")]
-atomic!(u64, "d");
-#[cfg(target_pointer_width = "32")]
-atomic!(isize, "w");
-#[cfg(target_pointer_width = "32")]
-atomic!(usize, "w");
-#[cfg(target_pointer_width = "64")]
-atomic!(isize, "d");
-#[cfg(target_pointer_width = "64")]
-atomic!(usize, "d");
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
diff --git a/src/arch_legacy/mod.rs b/src/arch_legacy/mod.rs
deleted file mode 100644
index d61382cd..00000000
--- a/src/arch_legacy/mod.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-// This module contains the atomic implementation for older rustc that does not support MaybeUninit registers.
-//
-// The implementation is based on the code just before we started using MaybeUninit registers.
-
-#![allow(missing_docs)] // For cfg_* macros.
-
-#[cfg(not(any(
- target_arch = "x86",
- target_arch = "x86_64",
- all(
- target_arch = "arm",
- any(
- target_feature = "v6",
- atomic_maybe_uninit_target_feature = "v6",
- target_os = "linux",
- target_os = "android",
- ),
- ),
- target_arch = "aarch64",
- target_arch = "riscv32",
- target_arch = "riscv64",
- all(target_arch = "loongarch64", not(atomic_maybe_uninit_no_asm)),
-)))]
-#[path = "../arch/unsupported.rs"]
-mod unsupported;
-
-#[cfg(target_arch = "aarch64")]
-mod aarch64;
-#[cfg(target_arch = "arm")]
-#[cfg(all(
- any(target_feature = "v6", atomic_maybe_uninit_target_feature = "v6"),
- not(any(
- target_feature = "v8",
- atomic_maybe_uninit_target_feature = "v8",
- target_feature = "v8m",
- atomic_maybe_uninit_target_feature = "v8m",
- )),
-))]
-mod arm;
-#[cfg(target_arch = "arm")]
-#[cfg(all(
- any(target_os = "linux", target_os = "android"),
- not(any(target_feature = "v6", atomic_maybe_uninit_target_feature = "v6")),
-))]
-mod arm_linux;
-#[cfg(target_arch = "arm")]
-#[cfg(any(
- target_feature = "v8",
- atomic_maybe_uninit_target_feature = "v8",
- target_feature = "v8m",
- atomic_maybe_uninit_target_feature = "v8m",
-))]
-mod armv8;
-#[cfg(target_arch = "loongarch64")]
-#[cfg(not(atomic_maybe_uninit_no_asm))]
-mod loongarch;
-#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
-mod riscv;
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-mod x86;
diff --git a/src/arch_legacy/riscv.rs b/src/arch_legacy/riscv.rs
deleted file mode 100644
index 8bea9b53..00000000
--- a/src/arch_legacy/riscv.rs
+++ /dev/null
@@ -1,460 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-RISC-V
-
-See arch/riscv.rs for references and notes.
-
-Generated asm:
-- riscv64gc https://godbolt.org/z/nW3Po8n4K
-- riscv32imac https://godbolt.org/z/51nPPMYze
-*/
-
-use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering};
-
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-use crate::raw::{AtomicCompareExchange, AtomicSwap};
-use crate::raw::{AtomicLoad, AtomicStore};
-
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-#[cfg(target_arch = "riscv32")]
-macro_rules! w {
- () => {
- ""
- };
-}
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-#[cfg(target_arch = "riscv64")]
-macro_rules! w {
- () => {
- "w"
- };
-}
-
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-macro_rules! atomic_rmw_amo {
- ($op:ident, $order:ident) => {
- match $order {
- Ordering::Relaxed => $op!(""),
- Ordering::Acquire => $op!(".aq"),
- Ordering::Release => $op!(".rl"),
- // AcqRel and SeqCst RMWs are equivalent.
- Ordering::AcqRel | Ordering::SeqCst => $op!(".aqrl"),
- _ => unreachable!(),
- }
- };
-}
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-macro_rules! atomic_rmw_lr_sc {
- ($op:ident, $order:ident) => {
- match $order {
- Ordering::Relaxed => $op!("", ""),
- Ordering::Acquire => $op!(".aq", ""),
- Ordering::Release => $op!("", ".rl"),
- Ordering::AcqRel => $op!(".aq", ".rl"),
- Ordering::SeqCst => $op!(".aqrl", ".rl"),
- _ => unreachable!(),
- }
- };
-}
-
-#[rustfmt::skip]
-macro_rules! atomic_load_store {
- ($ty:ident, $asm_suffix:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_load {
- ($acquire:tt, $release:tt) => {
- asm!(
- // (atomic) load from src to tmp
- $release,
- concat!("l", $asm_suffix, " {tmp}, 0({src})"),
- $acquire,
- // store tmp to out
- concat!("s", $asm_suffix, " {tmp}, 0({out})"),
- src = in(reg) ptr_reg!(src),
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_load!("", ""),
- Ordering::Acquire => atomic_load!("fence r, rw", ""),
- Ordering::SeqCst => atomic_load!("fence r, rw", "fence rw, rw"),
- _ => unreachable!(),
- }
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! atomic_store {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from val to tmp
- concat!("l", $asm_suffix, " {tmp}, 0({val})"),
- // (atomic) store tmp to dst
- $release,
- concat!("s", $asm_suffix, " {tmp}, 0({dst})"),
- $acquire,
- dst = inout(reg) ptr_reg!(dst) => _,
- val = in(reg) ptr_reg!(val),
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- match order {
- Ordering::Relaxed => atomic_store!("", ""),
- Ordering::Release => atomic_store!("", "fence rw, w"),
- // https://github.com/llvm/llvm-project/commit/3ea8f2526541884e03d5bd4f4e46f4eb190990b6
- Ordering::SeqCst => atomic_store!("fence rw, rw", "fence rw, w"),
- _ => unreachable!(),
- }
- }
- }
- }
- };
-}
-
-macro_rules! atomic {
- ($ty:ident, $asm_suffix:tt) => {
- atomic_load_store!($ty, $asm_suffix);
- #[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- macro_rules! swap {
- ($order:tt) => {
- asm!(
- // load from val (ptr) to val (val)
- concat!("l", $asm_suffix, " {val}, 0({val})"),
- // (atomic) swap (AMO)
- // - load value from dst and store it to tmp
- // - store value of val to dst
- concat!("amoswap.", $asm_suffix, $order, " {tmp}, {val}, 0({dst})"),
- // store tmp to out
- concat!("s", $asm_suffix, " {tmp}, 0({out})"),
- dst = in(reg) ptr_reg!(dst),
- val = inout(reg) ptr_reg!(val) => _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- tmp = lateout(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw_amo!(swap, order);
- }
- out
- }
- }
- #[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: crate::utils::RegSize;
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt) => {
- asm!(
- // load from old/new (ptr) to old/new (val)
- concat!("l", $asm_suffix, " {old}, 0({old})"),
- concat!("l", $asm_suffix, " {new}, 0({new})"),
- // (atomic) CAS (LR/SC loop)
- "2:",
- concat!("lr.", $asm_suffix, $acquire, " {tmp}, 0({dst})"),
- "bne {tmp}, {old}, 3f", // compare and jump if compare failed
- concat!("sc.", $asm_suffix, $release, " {r}, {new}, 0({dst})"),
- "bnez {r}, 2b", // continue loop if store failed
- "3:",
- "xor {r}, {tmp}, {old}",
- "seqz {r}, {r}",
- // store tmp to out
- concat!("s", $asm_suffix, " {tmp}, 0({out})"),
- dst = in(reg) ptr_reg!(dst),
- old = inout(reg) ptr_reg!(old) => _,
- new = inout(reg) ptr_reg!(new) => _,
- out = in(reg) ptr_reg!(out_ptr),
- tmp = out(reg) _,
- r = out(reg) r,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw_lr_sc!(cmpxchg, order);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-#[rustfmt::skip]
-macro_rules! atomic_sub_word {
- ($ty:ident, $asm_suffix:tt) => {
- atomic_load_store!($ty, $asm_suffix);
- #[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- // Implement sub-word atomic operations using word-sized LL/SC loop.
- // Based on assemblies generated by rustc/LLVM.
- // See also create_sub_word_mask_values.
- macro_rules! swap {
- ($acquire:tt, $release:tt) => {
- asm!(
- concat!("l", $asm_suffix, "u {val}, 0({val})"),
- concat!("sll", w!(), " {mask}, {mask}, {shift}"),
- concat!("sll", w!(), " {val}, {val}, {shift}"),
- // (atomic) swap (LR/SC loop)
- "2:",
- concat!("lr.w", $acquire, " {tmp1}, 0({dst})"),
- "mv {tmp2}, {val}",
- "xor {tmp2}, {tmp2}, {tmp1}",
- "and {tmp2}, {tmp2}, {mask}",
- "xor {tmp2}, {tmp2}, {tmp1}",
- concat!("sc.w", $release, " {tmp2}, {tmp2}, 0({dst})"),
- "bnez {tmp2}, 2b",
- concat!("srl", w!(), " {tmp1}, {tmp1}, {shift}"),
- concat!("s", $asm_suffix, " {tmp1}, 0({out})"),
- dst = in(reg) ptr_reg!(dst),
- val = inout(reg) ptr_reg!(val) => _,
- out = in(reg) ptr_reg!(out_ptr),
- shift = in(reg) shift,
- mask = inout(reg) mask => _,
- tmp1 = out(reg) _,
- tmp2 = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw_lr_sc!(swap, order);
- }
- out
- }
- }
- #[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- success: Ordering,
- failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let order = crate::utils::upgrade_success_ordering(success, failure);
- let (dst, shift, mask) = crate::utils::create_sub_word_mask_values(dst);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- let mut r: crate::utils::RegSize;
- // Implement sub-word atomic operations using word-sized LL/SC loop.
- // Based on assemblies generated by rustc/LLVM.
- // See also create_sub_word_mask_values.
- macro_rules! cmpxchg {
- ($acquire:tt, $release:tt) => {
- asm!(
- concat!("l", $asm_suffix, "u {old}, 0({old})"),
- concat!("l", $asm_suffix, "u {new}, 0({new})"),
- concat!("sll", w!(), " {mask}, {mask}, {shift}"),
- concat!("sll", w!(), " {old}, {old}, {shift}"),
- concat!("sll", w!(), " {new}, {new}, {shift}"),
- // (atomic) CAS (LR/SC loop)
- "2:",
- concat!("lr.w", $acquire, " {tmp1}, 0({dst})"),
- "and {tmp2}, {tmp1}, {mask}",
- "bne {tmp2}, {old}, 3f",
- "xor {tmp2}, {tmp1}, {new}",
- "and {tmp2}, {tmp2}, {mask}",
- "xor {tmp2}, {tmp2}, {tmp1}",
- concat!("sc.w", $release, " {tmp2}, {tmp2}, 0({dst})"),
- "bnez {tmp2}, 2b",
- "3:",
- concat!("srl", w!(), " {tmp2}, {tmp1}, {shift}"),
- "and {tmp1}, {tmp1}, {mask}",
- "xor {r}, {old}, {tmp1}",
- "seqz {r}, {r}",
- concat!("s", $asm_suffix, " {tmp2}, 0({out})"),
- dst = in(reg) ptr_reg!(dst),
- old = inout(reg) ptr_reg!(old) => _,
- new = inout(reg) ptr_reg!(new) => _,
- out = inout(reg) ptr_reg!(out_ptr) => _,
- shift = in(reg) shift,
- mask = inout(reg) mask => _,
- r = lateout(reg) r,
- tmp1 = out(reg) _,
- tmp2 = out(reg) _,
- options(nostack, preserves_flags),
- )
- };
- }
- atomic_rmw_lr_sc!(cmpxchg, order);
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-atomic_sub_word!(i8, "b");
-atomic_sub_word!(u8, "b");
-atomic_sub_word!(i16, "h");
-atomic_sub_word!(u16, "h");
-atomic!(i32, "w");
-atomic!(u32, "w");
-#[cfg(target_arch = "riscv64")]
-atomic!(i64, "d");
-#[cfg(target_arch = "riscv64")]
-atomic!(u64, "d");
-#[cfg(target_pointer_width = "32")]
-atomic!(isize, "w");
-#[cfg(target_pointer_width = "32")]
-atomic!(usize, "w");
-#[cfg(target_pointer_width = "64")]
-atomic!(isize, "d");
-#[cfg(target_pointer_width = "64")]
-atomic!(usize, "d");
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-#[cfg(target_arch = "riscv64")]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(target_arch = "riscv64")]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[cfg(target_arch = "riscv32")]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[cfg(target_arch = "riscv32")]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a"))]
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a")))]
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(any(target_feature = "a", atomic_maybe_uninit_target_feature = "a")))]
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
diff --git a/src/arch_legacy/x86.rs b/src/arch_legacy/x86.rs
deleted file mode 100644
index 6a9c7e91..00000000
--- a/src/arch_legacy/x86.rs
+++ /dev/null
@@ -1,856 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-/*
-x86 and x86_64
-
-See arch/x86.rs for references and notes.
-
-Generated asm:
-- x86_64 https://godbolt.org/z/fvqWGT5E6
-- x86_64 (+cmpxchg16b) https://godbolt.org/z/fGdj8naT9
-- x86 (i686) https://godbolt.org/z/9jKcboaoG
-- x86 (i686,-sse2) https://godbolt.org/z/sjYK57r96
-- x86 (i586) https://godbolt.org/z/5rrzYGxPe
-- x86 (i586,-x87) https://godbolt.org/z/GvcdhqxYo
-*/
-
-use core::{arch::asm, mem::MaybeUninit, sync::atomic::Ordering};
-
-use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
-
-#[cfg(target_pointer_width = "32")]
-macro_rules! ptr_modifier {
- () => {
- ":e"
- };
-}
-#[cfg(target_pointer_width = "64")]
-macro_rules! ptr_modifier {
- () => {
- ""
- };
-}
-
-#[cfg(target_arch = "x86")]
-#[cfg(not(atomic_maybe_uninit_no_cmpxchg8b))]
-#[cfg(target_feature = "sse")]
-#[cfg(target_feature = "sse2")]
-macro_rules! if_sse2 {
- ($then:expr, $else:expr) => {
- $then
- };
-}
-#[cfg(target_arch = "x86")]
-#[cfg(not(atomic_maybe_uninit_no_cmpxchg8b))]
-#[cfg(target_feature = "sse")]
-#[cfg(not(target_feature = "sse2"))]
-macro_rules! if_sse2 {
- ($then:expr, $else:expr) => {
- $else
- };
-}
-
-macro_rules! atomic {
- (
- $ty:ident, $val_reg:tt, $val_modifier:tt, $ptr_size:tt, $cmpxchg_cmp_reg:tt,
- $tmp_new_reg:tt
- ) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- // atomic load is always SeqCst.
- asm!(
- // (atomic) load from src to tmp
- concat!("mov {tmp", $val_modifier, "}, ", $ptr_size, " ptr [{src", ptr_modifier!(), "}]"),
- // store tmp to out
- concat!("mov ", $ptr_size, " ptr [{out", ptr_modifier!(), "}], {tmp", $val_modifier, "}"),
- src = in(reg) src,
- out = inout(reg) out_ptr => _,
- tmp = lateout($val_reg) _,
- options(nostack, preserves_flags),
- );
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- match order {
- // Relaxed and Release stores are equivalent.
- Ordering::Relaxed | Ordering::Release => {
- asm!(
- // load from val to tmp
- concat!("mov {tmp", $val_modifier, "}, ", $ptr_size, " ptr [{val", ptr_modifier!(), "}]"),
- // (atomic) store tmp to dst
- concat!("mov ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {tmp", $val_modifier, "}"),
- dst = inout(reg) dst => _,
- val = in(reg) val,
- tmp = lateout($val_reg) _,
- options(nostack, preserves_flags),
- );
- }
- Ordering::SeqCst => {
- asm!(
- // load from val to tmp
- concat!("mov {tmp", $val_modifier, "}, ", $ptr_size, " ptr [{val", ptr_modifier!(), "}]"),
- // (atomic) store tmp to dst (SeqCst store is xchg, not mov)
- concat!("xchg ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {tmp", $val_modifier, "}"),
- dst = inout(reg) dst => _,
- val = in(reg) val,
- tmp = lateout($val_reg) _,
- options(nostack, preserves_flags),
- );
- }
- _ => unreachable!(),
- }
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- unsafe {
- // atomic swap is always SeqCst.
- asm!(
- // load from val to tmp
- concat!("mov {tmp", $val_modifier, "}, ", $ptr_size, " ptr [{val", ptr_modifier!(), "}]"),
- // (atomic) swap tmp and dst
- concat!("xchg ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], {tmp", $val_modifier, "}"),
- // store tmp to out
- concat!("mov ", $ptr_size, " ptr [{out", ptr_modifier!(), "}], {tmp", $val_modifier, "}"),
- dst = inout(reg) dst => _,
- val = in(reg) val,
- out = inout(reg) out_ptr => _,
- tmp = lateout($val_reg) _,
- options(nostack, preserves_flags),
- );
- }
- out
- }
- }
- #[cfg(not(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg)))]
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- _failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg
- unsafe {
- let mut r: u8;
- // compare_exchange is always SeqCst.
- asm!(
- // load from old/new to $cmpxchg_cmp_reg/$tmp_new_reg
- concat!("mov ", $cmpxchg_cmp_reg, ", ", $ptr_size, " ptr [{old", ptr_modifier!(), "}]"),
- concat!("mov ", $tmp_new_reg, ", ", $ptr_size, " ptr [{new", ptr_modifier!(), "}]"),
- // (atomic) CAS
- // - Compare $cmpxchg_cmp_reg with dst.
- // - If equal, ZF is set and $tmp_new_reg is loaded into dst.
- // - Else, clear ZF and load dst into $cmpxchg_cmp_reg.
- concat!("lock cmpxchg ", $ptr_size, " ptr [{dst", ptr_modifier!(), "}], ", $tmp_new_reg),
- // load ZF to cl
- "sete cl",
- // store $cmpxchg_cmp_reg to out
- concat!("mov ", $ptr_size, " ptr [{out", ptr_modifier!(), "}], ", $cmpxchg_cmp_reg),
- dst = in(reg) dst,
- old = in(reg) old,
- new = in(reg) new,
- out = in(reg) out_ptr,
- out("cl") r,
- out($cmpxchg_cmp_reg) _,
- // Do not use `preserves_flags` because CMPXCHG modifies the ZF flag.
- options(nostack),
- );
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-atomic!(i8, reg_byte, "", "byte", "al", "cl");
-atomic!(u8, reg_byte, "", "byte", "al", "cl");
-atomic!(i16, reg, ":x", "word", "ax", "cx");
-atomic!(u16, reg, ":x", "word", "ax", "cx");
-atomic!(i32, reg, ":e", "dword", "eax", "ecx");
-atomic!(u32, reg, ":e", "dword", "eax", "ecx");
-#[cfg(target_arch = "x86_64")]
-atomic!(i64, reg, "", "qword", "rax", "rcx");
-#[cfg(target_arch = "x86_64")]
-atomic!(u64, reg, "", "qword", "rax", "rcx");
-#[cfg(target_pointer_width = "32")]
-atomic!(isize, reg, ":e", "dword", "eax", "ecx");
-#[cfg(target_pointer_width = "32")]
-atomic!(usize, reg, ":e", "dword", "eax", "ecx");
-#[cfg(target_pointer_width = "64")]
-atomic!(isize, reg, "", "qword", "rax", "rcx");
-#[cfg(target_pointer_width = "64")]
-atomic!(usize, reg, "", "qword", "rax", "rcx");
-
-// For load/store, we can use MOVQ(SSE2)/MOVLPS(SSE) instead of CMPXCHG8B.
-// Refs: https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0/llvm/test/CodeGen/X86/atomic-load-store-wide.ll
-#[cfg(target_arch = "x86")]
-#[cfg(not(atomic_maybe_uninit_no_cmpxchg8b))]
-macro_rules! atomic64 {
- ($ty:ident) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- #[cfg(target_feature = "sse")]
- // SAFETY: the caller must uphold the safety contract.
- // cfg guarantees that the CPU supports SSE.
- unsafe {
- #[cfg(target_feature = "sse2")]
- {
- // atomic load is always SeqCst.
- asm!(
- // Refs:
- // - https://www.felixcloutier.com/x86/movq (SSE2)
- // - https://www.felixcloutier.com/x86/movd:movq (SSE2)
- // - https://www.felixcloutier.com/x86/pshufd (SSE2)
- // (atomic) load from src to tmp0
- "movq {tmp0}, qword ptr [{src}]",
- // extract lower 64-bits
- "pshufd {tmp1}, {tmp0}, 85",
- // store tmp0/tmp1 to out
- "movd dword ptr [{out}], {tmp0}",
- "movd dword ptr [{out} + 4], {tmp1}",
- src = in(reg) src,
- out = in(reg) out_ptr,
- tmp0 = out(xmm_reg) _,
- tmp1 = out(xmm_reg) _,
- options(nostack, preserves_flags),
- );
- }
- #[cfg(not(target_feature = "sse2"))]
- {
- // atomic load is always SeqCst.
- asm!(
- // Refs:
- // - https://www.felixcloutier.com/x86/xorps (SSE)
- // - https://www.felixcloutier.com/x86/movlps (SSE)
- // - https://www.felixcloutier.com/x86/movss (SSE)
- // - https://www.felixcloutier.com/x86/shufps (SSE)
- "xorps {tmp}, {tmp}",
- // (atomic) load from src to tmp
- "movlps {tmp}, qword ptr [{src}]",
- // store tmp to out
- "movss dword ptr [{out}], {tmp}",
- "shufps {tmp}, {tmp}, 85",
- "movss dword ptr [{out} + 4], {tmp}",
- src = in(reg) src,
- out = in(reg) out_ptr,
- tmp = out(xmm_reg) _,
- options(nostack, preserves_flags),
- );
- }
- }
- #[cfg(not(target_feature = "sse"))]
- // SAFETY: the caller must uphold the safety contract.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- // atomic load is always SeqCst.
- asm!(
- "xchg {esi_tmp}, esi", // save esi which is reserved by LLVM
- // (atomic) load by cmpxchg(0, 0)
- "lock cmpxchg8b qword ptr [edi]",
- // store current value to out
- "mov dword ptr [esi], eax",
- "mov dword ptr [esi + 4], edx",
- "mov esi, {esi_tmp}", // restore esi
- esi_tmp = inout(reg) out_ptr => _,
- // set old/new args of cmpxchg8b to 0
- inout("eax") 0_u32 => _,
- inout("edx") 0_u32 => _,
- in("ebx") 0_u32,
- in("ecx") 0_u32,
- in("edi") src,
- // Do not use `preserves_flags` because CMPXCHG8B modifies the ZF flag.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- #[cfg(target_feature = "sse")]
- // SAFETY: the caller must uphold the safety contract.
- // cfg guarantees that the CPU supports SSE.
- //
- // Refs:
- // - https://www.felixcloutier.com/x86/movlps (SSE)
- // - https://www.felixcloutier.com/x86/xorps (SSE)
- // - https://www.felixcloutier.com/x86/movsd (SSE2)
- // - https://www.felixcloutier.com/x86/lock
- // - https://www.felixcloutier.com/x86/or
- unsafe {
- match order {
- // Relaxed and Release stores are equivalent.
- Ordering::Relaxed | Ordering::Release => {
- asm!(
- if_sse2!("", "xorps {tmp}, {tmp}"),
- // load from val to tmp
- if_sse2!("movsd {tmp}, qword ptr [{val}]", "movlps {tmp}, qword ptr [{val}]"),
- // (atomic) store tmp to dst
- "movlps qword ptr [{dst}], {tmp}",
- dst = in(reg) dst,
- val = in(reg) val,
- tmp = out(xmm_reg) _,
- options(nostack, preserves_flags),
- );
- }
- Ordering::SeqCst => {
- asm!(
- // load from val to tmp
- if_sse2!("", "xorps {tmp}, {tmp}"),
- if_sse2!("movsd {tmp}, qword ptr [{val}]", "movlps {tmp}, qword ptr [{val}]"),
- // (atomic) store tmp to dst
- "movlps qword ptr [{dst}], {tmp}",
- "lock or dword ptr [esp], 0", // equivalent to mfence, but doesn't require SSE2
- dst = in(reg) dst,
- val = in(reg) val,
- tmp = out(xmm_reg) _,
- // Do not use `preserves_flags` because OR modifies the OF, CF, SF, ZF, and PF flags.
- options(nostack),
- );
- }
- _ => unreachable!(),
- }
- }
- #[cfg(not(target_feature = "sse"))]
- // SAFETY: the caller must uphold the safety contract.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- // atomic store is always SeqCst.
- let _ = order;
- asm!(
- "mov ebx, dword ptr [eax]",
- "mov ecx, dword ptr [eax + 4]",
- // This is based on the code generated for the first load in DW RMWs by LLVM,
- // but it is interesting that they generate code that does mixed-sized atomic access.
- //
- // This is not single-copy atomic reads, but this is ok because subsequent
- // CAS will check for consistency.
- "mov eax, dword ptr [edi]",
- "mov edx, dword ptr [edi + 4]",
- // (atomic) store (CAS loop)
- "2:",
- "lock cmpxchg8b qword ptr [edi]",
- "jne 2b",
- inout("eax") val => _,
- out("edx") _,
- out("ebx") _,
- out("ecx") _,
- in("edi") dst,
- // Do not use `preserves_flags` because CMPXCHG8B modifies the ZF flag.
- options(nostack),
- );
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- // atomic store is always SeqCst.
- asm!(
- "xchg {esi_tmp}, esi", // save esi which is reserved by LLVM
- "mov ebx, dword ptr [eax]",
- "mov ecx, dword ptr [eax + 4]",
- // This is based on the code generated for the first load in DW RMWs by LLVM,
- // but it is interesting that they generate code that does mixed-sized atomic access.
- //
- // This is not single-copy atomic reads, but this is ok because subsequent
- // CAS will check for consistency.
- "mov eax, dword ptr [edi]",
- "mov edx, dword ptr [edi + 4]",
- // (atomic) swap (CAS loop)
- "2:",
- "lock cmpxchg8b qword ptr [edi]",
- "jne 2b",
- // store previous value to out
- "mov dword ptr [esi], eax",
- "mov dword ptr [esi + 4], edx",
- "mov esi, {esi_tmp}", // restore esi
- esi_tmp = inout(reg) out_ptr => _,
- inout("eax") val => _,
- out("edx") _,
- out("ebx") _,
- out("ecx") _,
- in("edi") dst,
- // Do not use `preserves_flags` because CMPXCHG8B modifies the ZF flag.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- _failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must uphold the safety contract.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg
- unsafe {
- let mut r: u32;
- // compare_exchange is always SeqCst.
- asm!(
- "xchg {esi_tmp}, esi", // save esi which is reserved by LLVM
- "mov eax, dword ptr [edx]",
- "mov edx, dword ptr [edx + 4]",
- "mov ebx, dword ptr [ecx]",
- "mov ecx, dword ptr [ecx + 4]",
- // (atomic) CAS
- "lock cmpxchg8b qword ptr [edi]",
- "sete cl",
- // store previous value to out
- "mov dword ptr [esi], eax",
- "mov dword ptr [esi + 4], edx",
- "mov esi, {esi_tmp}", // restore esi
- esi_tmp = inout(reg) out_ptr => _,
- out("eax") _,
- inout("edx") old => _,
- out("ebx") _,
- inout("ecx") new => r,
- in("edi") dst,
- // Do not use `preserves_flags` because CMPXCHG8B modifies the ZF flag.
- options(nostack),
- );
- let r = r.to_ne_bytes()[0];
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-#[cfg(target_arch = "x86")]
-#[cfg(not(atomic_maybe_uninit_no_cmpxchg8b))]
-atomic64!(i64);
-#[cfg(target_arch = "x86")]
-#[cfg(not(atomic_maybe_uninit_no_cmpxchg8b))]
-atomic64!(u64);
-
-#[cfg(target_arch = "x86_64")]
-#[cfg(any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"))]
-macro_rules! atomic128 {
- ($ty:ident) => {
- #[cfg(target_pointer_width = "32")]
- atomic128!($ty, "edi", "esi", "r8d", "edx");
- #[cfg(target_pointer_width = "64")]
- atomic128!($ty, "rdi", "rsi", "r8", "rdx");
- };
- ($ty:ident, $rdi:tt, $rsi:tt, $r8:tt, $rdx:tt) => {
- impl AtomicLoad for $ty {
- #[inline]
- unsafe fn atomic_load(
- src: *const MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(src, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
-
- // SAFETY: the caller must guarantee that `src` is valid for both writes and
- // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
- // cfg guarantees that the CPU supports CMPXCHG16B.
- //
- // If the value at `dst` (destination operand) and rdx:rax are equal, the
- // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at
- // `dst` is loaded to rdx:rax.
- //
- // The ZF flag is set if the value at `dst` and rdx:rax are equal,
- // otherwise it is cleared. Other flags are unaffected.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- // atomic load is always SeqCst.
- asm!(
- "mov {rbx_tmp}, rbx", // save rbx which is reserved by LLVM
- "xor rbx, rbx", // zeroed rbx
- // (atomic) load by cmpxchg(0, 0)
- concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
- // store current value to out
- concat!("mov qword ptr [", $rsi, "], rax"),
- concat!("mov qword ptr [", $rsi, " + 8], rdx"),
- "mov rbx, {rbx_tmp}", // restore rbx
- // set old/new args of cmpxchg16b to 0 (rbx is zeroed after saved to rbx_tmp, to avoid xchg)
- rbx_tmp = out(reg) _,
- in("rcx") 0_u64,
- inout("rax") 0_u64 => _,
- inout("rdx") 0_u64 => _,
- in($rdi) src,
- in($rsi) out_ptr,
- // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicStore for $ty {
- #[inline]
- unsafe fn atomic_store(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let val = val.as_ptr();
-
- // SAFETY: the caller must guarantee that `dst` is valid for both writes and
- // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
- // cfg guarantees that the CPU supports CMPXCHG16B.
- //
- // If the value at `dst` (destination operand) and rdx:rax are equal, the
- // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at
- // `dst` is loaded to rdx:rax.
- //
- // The ZF flag is set if the value at `dst` and rdx:rax are equal,
- // otherwise it is cleared. Other flags are unaffected.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- // atomic store is always SeqCst.
- asm!(
- "mov {rbx_tmp}, rbx", // save rbx which is reserved by LLVM
- concat!("mov rbx, qword ptr [", $rsi, "]"),
- concat!("mov rcx, qword ptr [", $rsi, " + 8]"),
- // This is based on the code generated for the first load in DW RMWs by LLVM,
- // but it is interesting that they generate code that does mixed-sized atomic access.
- //
- // This is not single-copy atomic reads, but this is ok because subsequent
- // CAS will check for consistency.
- concat!("mov rax, qword ptr [", $rdi, "]"),
- concat!("mov rdx, qword ptr [", $rdi, " + 8]"),
- // (atomic) store (CAS loop)
- "2:",
- concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
- "jne 2b",
- "mov rbx, {rbx_tmp}", // restore rbx
- rbx_tmp = out(reg) _,
- out("rax") _,
- out("rcx") _,
- out("rdx") _,
- in($rdi) dst,
- in($rsi) val,
- // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
- options(nostack),
- );
- }
- }
- }
- impl AtomicSwap for $ty {
- #[inline]
- unsafe fn atomic_swap(
- dst: *mut MaybeUninit,
- val: MaybeUninit,
- _order: Ordering,
- ) -> MaybeUninit {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let val = val.as_ptr();
-
- // SAFETY: the caller must guarantee that `dst` is valid for both writes and
- // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
- // cfg guarantees that the CPU supports CMPXCHG16B.
- //
- // If the value at `dst` (destination operand) and rdx:rax are equal, the
- // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at
- // `dst` is loaded to rdx:rax.
- //
- // The ZF flag is set if the value at `dst` and rdx:rax are equal,
- // otherwise it is cleared. Other flags are unaffected.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- // atomic swap is always SeqCst.
- asm!(
- "mov {rbx_tmp}, rbx", // save rbx which is reserved by LLVM
- concat!("mov rbx, qword ptr [", $rsi, "]"),
- concat!("mov rcx, qword ptr [", $rsi, " + 8]"),
- // This is based on the code generated for the first load in DW RMWs by LLVM,
- // but it is interesting that they generate code that does mixed-sized atomic access.
- //
- // This is not single-copy atomic reads, but this is ok because subsequent
- // CAS will check for consistency.
- concat!("mov rax, qword ptr [", $rdi, "]"),
- concat!("mov rdx, qword ptr [", $rdi, " + 8]"),
- // (atomic) swap (CAS loop)
- "2:",
- concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
- "jne 2b",
- // store previous value to out
- concat!("mov qword ptr [", $r8, "], rax"),
- concat!("mov qword ptr [", $r8, " + 8], rdx"),
- "mov rbx, {rbx_tmp}", // restore rbx
- rbx_tmp = out(reg) _,
- out("rax") _,
- out("rcx") _,
- out("rdx") _,
- in($rdi) dst,
- in($rsi) val,
- in($r8) out_ptr,
- // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
- options(nostack),
- );
- }
- out
- }
- }
- impl AtomicCompareExchange for $ty {
- #[inline]
- unsafe fn atomic_compare_exchange(
- dst: *mut MaybeUninit,
- old: MaybeUninit,
- new: MaybeUninit,
- _success: Ordering,
- _failure: Ordering,
- ) -> (MaybeUninit, bool) {
- debug_assert_atomic_unsafe_precondition!(dst, $ty);
- let mut out: MaybeUninit = MaybeUninit::uninit();
- let out_ptr = out.as_mut_ptr();
- let old = old.as_ptr();
- let new = new.as_ptr();
-
- // SAFETY: the caller must guarantee that `dst` is valid for both writes and
- // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
- // cfg guarantees that the CPU supports CMPXCHG16B.
- //
- // If the value at `dst` (destination operand) and rdx:rax are equal, the
- // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at
- // `dst` is loaded to rdx:rax.
- //
- // The ZF flag is set if the value at `dst` and rdx:rax are equal,
- // otherwise it is cleared. Other flags are unaffected.
- //
- // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
- unsafe {
- let mut r: u64;
- // compare_exchange is always SeqCst.
- asm!(
- "mov {rbx_tmp}, rbx", // save rbx which is reserved by LLVM
- concat!("mov rax, qword ptr [", $rsi, "]"),
- concat!("mov rsi, qword ptr [", $rsi, " + 8]"),
- concat!("mov rbx, qword ptr [", $rdx, "]"),
- concat!("mov rcx, qword ptr [", $rdx, " + 8]"),
- "mov rdx, rsi",
- // (atomic) CAS
- concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
- "sete cl",
- // store previous value to out
- concat!("mov qword ptr [", $r8, "], rax"),
- concat!("mov qword ptr [", $r8, " + 8], rdx"),
- "mov rbx, {rbx_tmp}", // restore rbx
- rbx_tmp = out(reg) _,
- out("rax") _,
- out("rcx") r,
- lateout("rdx") _,
- lateout("rsi") _,
- in($rdi) dst,
- in($rsi) old,
- in($rdx) new,
- in($r8) out_ptr,
- // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
- options(nostack),
- );
- let r = r.to_ne_bytes()[0];
- crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
- (out, r != 0)
- }
- }
- }
- };
-}
-
-#[cfg(target_arch = "x86_64")]
-#[cfg(any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"))]
-atomic128!(i128);
-#[cfg(target_arch = "x86_64")]
-#[cfg(any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"))]
-atomic128!(u128);
-
-// -----------------------------------------------------------------------------
-// cfg macros
-
-#[macro_export]
-macro_rules! cfg_has_atomic_8 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_8 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_16 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_16 {
- ($($tt:tt)*) => {};
-}
-#[macro_export]
-macro_rules! cfg_has_atomic_32 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[macro_export]
-macro_rules! cfg_no_atomic_32 {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg8b)))]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg8b)))]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[cfg(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg8b))]
-#[macro_export]
-macro_rules! cfg_has_atomic_64 {
- ($($tt:tt)*) => {};
-}
-#[cfg(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg8b))]
-#[macro_export]
-macro_rules! cfg_no_atomic_64 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(all(
- target_arch = "x86_64",
- any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"),
-)))]
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(all(
- target_arch = "x86_64",
- any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"),
-)))]
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(all(
- target_arch = "x86_64",
- any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"),
-))]
-#[macro_export]
-macro_rules! cfg_has_atomic_128 {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(all(
- target_arch = "x86_64",
- any(target_feature = "cmpxchg16b", atomic_maybe_uninit_target_feature = "cmpxchg16b"),
-))]
-#[macro_export]
-macro_rules! cfg_no_atomic_128 {
- ($($tt:tt)*) => {};
-}
-#[cfg(not(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg)))]
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
-#[cfg(not(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg)))]
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => {};
-}
-#[cfg(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg))]
-#[macro_export]
-macro_rules! cfg_has_atomic_cas {
- ($($tt:tt)*) => {};
-}
-#[cfg(all(target_arch = "x86", atomic_maybe_uninit_no_cmpxchg))]
-#[macro_export]
-macro_rules! cfg_no_atomic_cas {
- ($($tt:tt)*) => { $($tt)* };
-}
diff --git a/src/gen/utils.rs b/src/gen/utils.rs
index 53023068..51da45c5 100644
--- a/src/gen/utils.rs
+++ b/src/gen/utils.rs
@@ -49,20 +49,12 @@ mod imp {
macro_rules! ptr_reg {
($ptr:ident) => {{
let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)
- #[cfg(not(atomic_maybe_uninit_no_asm_maybe_uninit))]
#[allow(clippy::ptr_as_ptr)]
{
// If we cast to u64 here, the provenance will be lost,
// so we convert to MaybeUninit via zero extend helper.
crate::utils::zero_extend64_ptr($ptr as *mut ())
}
- #[cfg(atomic_maybe_uninit_no_asm_maybe_uninit)]
- {
- // Use cast on old rustc because it does not support MaybeUninit
- // registers. This is still permissive-provenance compatible and
- // is sound.
- $ptr as u64
- }
}};
}
pub(crate) type RegSize = u64;
diff --git a/src/lib.rs b/src/lib.rs
index 55fdcb8b..8acf76fa 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -29,10 +29,10 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch, Arm64EC, s390x, MIPS, P
| riscv32 (+zacas) \[4] | i64,u64 | ✓ | ✓ |
| riscv64 | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓\[1] |
| riscv64 (+zacas) \[4] | i128,u128 | ✓ | ✓ |
-| loongarch64 \[7] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ |
-| loongarch32 \[10] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
-| arm64ec \[8] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
-| s390x \[8] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
+| loongarch64 | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ |
+| loongarch32 \[8] (experimental) | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
+| arm64ec \[7] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
+| s390x \[7] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64,i128,u128 | ✓ | ✓ |
| mips / mips32r6 \[9] | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
| mips64 / mips64r6 \[9] | isize,usize,i8,u8,i16,u16,i32,u32,i64,u64 | ✓ | ✓ |
| powerpc \[9] | isize,usize,i8,u8,i16,u16,i32,u32 | ✓ | ✓ |
@@ -52,9 +52,8 @@ Currently, x86, x86_64, Arm, AArch64, RISC-V, LoongArch, Arm64EC, s390x, MIPS, P
\[4] Requires `zacas` target feature.
\[5] Requires `quadword-atomics` target feature (enabled by default on powerpc64le).
\[6] Requires `v9` or `leoncasa` target feature (enabled by default on Linux).
-\[7] Requires Rust 1.72+.
-\[8] Requires Rust 1.84+.
-\[10] Requires Rust 1.91+.
+\[7] Requires Rust 1.84+.
+\[8] Requires Rust 1.91+.
\[9] Requires nightly due to `#![feature(asm_experimental_arch)]`.
See also [Atomic operation overview by architecture](https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md) for more information about atomic operations in these architectures.
@@ -81,7 +80,6 @@ Feel free to submit an issue if your target is not supported yet.
allow(dead_code, unused_variables)
)
))]
-#![warn(unsafe_op_in_unsafe_fn)]
#![warn(
// Lints that may help when writing public library.
missing_debug_implementations,
@@ -112,7 +110,6 @@ mod utils;
#[macro_use]
mod tests;
-#[cfg_attr(atomic_maybe_uninit_no_asm_maybe_uninit, path = "arch_legacy/mod.rs")]
mod arch;
pub mod raw;
@@ -169,29 +166,24 @@ unsafe impl Sync for AtomicMaybeUninit {}
impl core::panic::RefUnwindSafe for AtomicMaybeUninit {}
impl AtomicMaybeUninit {
- const_fn! {
- const_if: #[cfg(not(atomic_maybe_uninit_no_const_fn_trait_bound))];
- /// Creates a new atomic value from a potentially uninitialized value.
- ///
- /// This is `const fn` on Rust 1.61+. See also `const_new` function, which is always `const fn`.
- ///
- /// # Examples
- ///
- /// ```
- /// use std::mem::MaybeUninit;
- ///
- /// use atomic_maybe_uninit::AtomicMaybeUninit;
- ///
- /// let v = AtomicMaybeUninit::new(MaybeUninit::new(5_i32));
- ///
- /// // Equivalent to:
- /// let v = AtomicMaybeUninit::from(5_i32);
- /// ```
- #[inline]
- #[must_use]
- pub const fn new(v: MaybeUninit) -> Self {
- Self { v: UnsafeCell::new(v), _align: [] }
- }
+ /// Creates a new atomic value from a potentially uninitialized value.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use std::mem::MaybeUninit;
+ ///
+ /// use atomic_maybe_uninit::AtomicMaybeUninit;
+ ///
+ /// let v = AtomicMaybeUninit::new(MaybeUninit::new(5_i32));
+ ///
+ /// // Equivalent to:
+ /// let v = AtomicMaybeUninit::from(5_i32);
+ /// ```
+ #[inline]
+ #[must_use]
+ pub const fn new(v: MaybeUninit) -> Self {
+ Self { v: UnsafeCell::new(v), _align: [] }
}
// TODO: update docs based on https://github.com/rust-lang/rust/pull/116762
@@ -256,30 +248,25 @@ impl AtomicMaybeUninit {
}
}
- const_fn! {
- const_if: #[cfg(not(atomic_maybe_uninit_no_const_fn_trait_bound))];
- /// Consumes the atomic and returns the contained value.
- ///
- /// This is safe because passing `self` by value guarantees that no other threads are
- /// concurrently accessing the atomic data.
- ///
- /// This is `const fn` on Rust 1.61+.
- ///
- /// # Examples
- ///
- /// ```
- /// use atomic_maybe_uninit::AtomicMaybeUninit;
- ///
- /// let v = AtomicMaybeUninit::from(5_i32);
- /// unsafe { assert_eq!(v.into_inner().assume_init(), 5) }
- /// ```
- #[inline]
- pub const fn into_inner(self) -> MaybeUninit {
- // SAFETY: AtomicMaybeUninit and MaybeUninit have the same size
- // and in-memory representations, so they can be safely transmuted.
- // (Equivalent to UnsafeCell::into_inner which is unstable in const context.)
- unsafe { utils::transmute_copy_by_val::