From c8528142a48ace20ca83dfb0529d3d98bf29e6ae Mon Sep 17 00:00:00 2001 From: EFanZh Date: Wed, 7 Oct 2026 21:17:08 +0800 Subject: [PATCH 1/4] Make `Rc::deref` zero-cost --- library/alloc/src/lib.rs | 2 + library/alloc/src/rcs/arc.rs | 8 +- library/alloc/src/rcs/common/mod.rs | 544 ++++++++++ library/alloc/src/rcs/common/rc_layout.rs | 249 +++++ .../alloc/src/rcs/common/rc_value_pointer.rs | 106 ++ library/alloc/src/rcs/rc.rs | 971 +++++------------- src/etc/gdb_providers.py | 53 +- src/etc/lldb_providers.py | 96 +- src/etc/natvis/liballoc.natvis | 49 +- src/tools/miri/tests/fail/memleak_rc.stderr | 25 +- .../lib-optimizations/rc-arc-optimizations.rs | 28 + tests/codegen-llvm/placement-new.rs | 6 +- tests/debuginfo/strings-and-strs.rs | 4 +- 13 files changed, 1339 insertions(+), 802 deletions(-) create mode 100644 library/alloc/src/rcs/common/mod.rs create mode 100644 library/alloc/src/rcs/common/rc_layout.rs create mode 100644 library/alloc/src/rcs/common/rc_value_pointer.rs create mode 100644 tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 4fb34ed5b2668..fa30885a0218c 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -242,6 +242,8 @@ mod raw_vec; /// Implementations of reference-counted pointers. #[cfg(not(no_rc))] mod rcs { + mod common; + pub mod rc; #[cfg(all(not(no_sync), target_has_atomic = "ptr"))] diff --git a/library/alloc/src/rcs/arc.rs b/library/alloc/src/rcs/arc.rs index 8ae3c094b2910..9abfa8f8a862d 100644 --- a/library/alloc/src/rcs/arc.rs +++ b/library/alloc/src/rcs/arc.rs @@ -29,7 +29,6 @@ use crate::alloc::{ }; use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; -use crate::rc::is_dangling; #[cfg(not(no_global_oom_handling))] use crate::string::String; #[cfg(not(no_global_oom_handling))] @@ -46,6 +45,13 @@ use crate::vec::Vec; /// See comment in `Arc::clone`. const MAX_REFCOUNT: usize = (isize::MAX) as usize; +fn is_dangling(ptr: *const T) -> bool +where + T: ?Sized, +{ + ptr.addr() == usize::MAX +} + #[cold] #[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[cfg_attr(panic = "immediate-abort", inline)] diff --git a/library/alloc/src/rcs/common/mod.rs b/library/alloc/src/rcs/common/mod.rs new file mode 100644 index 0000000000000..bf403a16e396f --- /dev/null +++ b/library/alloc/src/rcs/common/mod.rs @@ -0,0 +1,544 @@ +//! Provides infrastructure for reference counted types. +//! +//! A reference counted allocation contains the following parts in order: +//! +//! - Padding: allows header and value to be arranged consecutively while being properly aligned. +//! We choose the padding size to be +//! `size_of::
().next_multiple_of(align_of::()) - size_of::
()`. +//! - Header: stores reference counters. +//! - Value: the actual contained value. +//! +//! We use the following layout for the allocation: +//! +//! - Alignment: `align_of::
().max(align_of::())`. +//! - Size: `size_of::
().next_multiple_of(align_of::()) + size_of::()`. +//! +//! Note that it is possible that the size of an allocation is not a multiple of its alignment. + +use core::alloc::{AllocError, Allocator}; +use core::clone::CloneToUninit; +#[cfg(not(no_global_oom_handling))] +use core::iter::TrustedLen; +use core::mem::DropGuard; +#[cfg(not(no_global_oom_handling))] +use core::mem::ManuallyDrop; +use core::ptr; +#[cfg(not(no_global_oom_handling))] +use core::ptr::NonNull; + +pub(super) use rc_value_pointer::{ErasedRcValuePointer, RcValuePointer}; + +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{self, Global}; +#[cfg(not(no_global_oom_handling))] +use crate::boxed::Box; +use crate::rcs::common::rc_layout::RcLayout; +#[cfg(not(no_global_oom_handling))] +use crate::vec::Vec; + +mod rc_layout; +mod rc_value_pointer; + +// - `repr(C)` is required for deterministic reference counter layout, which is required in debugger +// visualizers. +// - `align(2)` is required to ensure `usize::MAX` can be used as the dangling pointer address. +#[repr(C, align(2))] +pub(super) struct Header { + pub(super) strong: C, + pub(super) weak: C, +} + +impl Header { + #[inline] + pub(super) fn try_allocate_uninit_for_type( + alloc: &A, + ) -> Result, AllocError> + where + A: Allocator, + { + Ok(try_allocate_uninit(alloc, RcLayout::new::())?.unerase()) + } + + #[inline] + pub(super) fn try_allocate_zeroed_for_type( + alloc: &A, + ) -> Result, AllocError> + where + A: Allocator, + { + Ok(try_allocate_zeroed(alloc, RcLayout::new::())?.unerase()) + } + + pub(super) fn try_allocate_from_cloning( + alloc: &A, + value: &T, + ) -> Result, AllocError> + where + A: Allocator, + T: CloneToUninit + ?Sized, + { + let Ok(rc_layout) = RcLayout::try_from_value::(value) else { + return Err(AllocError); + }; + + // SAFETY: `ptr` is allocated with layout calculated from `value`, we are safe to clone into it. + let ptr = try_allocate_with(alloc, rc_layout, |ptr| unsafe { + value.clone_to_uninit(ptr.ptr.cast().as_ptr()); + })?; + + Ok(ptr.unerase_with(ptr::metadata(value))) + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn allocate_uninit_for_type(alloc: &A) -> RcValuePointer + where + A: Allocator, + { + allocate_uninit(alloc, RcLayout::new::()).unerase() + } + + /// # Safety + /// + /// Caller must ensure that `T` is small enough to store inside a reference counted allocation + /// with header of type `Self`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) unsafe fn allocate_uninit_for_value_unchecked( + alloc: &A, + value: &T, + ) -> RcValuePointer + where + A: Allocator, + T: ?Sized, + { + // SAFETY: Caller guarantees we are safe to calculate the allocation layout. + allocate_uninit(alloc, unsafe { RcLayout::from_value_unchecked::(value) }) + .unerase_with(ptr::metadata(value)) + } + + #[cfg(not(no_global_oom_handling))] + pub(super) fn allocate_uninit_slice(alloc: &A, length: usize) -> RcValuePointer<[T]> + where + A: Allocator, + { + allocate_uninit(alloc, RcLayout::new_array::(length)).unerase_with(length) + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn allocate_zeroed_for_type(alloc: &A) -> RcValuePointer + where + A: Allocator, + { + allocate_zeroed(alloc, RcLayout::new::()).unerase() + } + + #[cfg(not(no_global_oom_handling))] + pub(super) fn allocate_zeroed_slice(alloc: &A, length: usize) -> RcValuePointer<[T]> + where + A: Allocator, + { + allocate_zeroed(alloc, RcLayout::new_array::(length)).unerase_with(length) + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn allocate_from_cloning(alloc: &A, value: &T) -> RcValuePointer + where + A: Allocator, + T: CloneToUninit + ?Sized, + { + // SAFETY: `ptr` is allocated with layout calculated from `value`, we are safe to clone into it. + let ptr = allocate_with(alloc, RcLayout::from_value::(value), |ptr| unsafe { + value.clone_to_uninit(ptr.ptr.cast().as_ptr()) + }); + + ptr.unerase_with(ptr::metadata(value)) + } + + #[cfg(not(no_global_oom_handling))] + pub(super) fn allocate_from_default(alloc: &A) -> RcValuePointer + where + A: Allocator, + T: Default, + { + // SAFETY: We have exclusive access to `ptr`. + allocate_with(alloc, RcLayout::new::(), |ptr| unsafe { + ptr.unerase().ptr.write(T::default()) + }) + .unerase() + } + + /// Allocates a reference-counted memory chunk for storing a value according to `rc_layout`, + /// then initializes the value by copying from `value`. Note that the value bytes are duplicated + /// but we don't require `T` to be `Copy`, the caller needs to prevent leaks and double-frees + /// manually. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn allocate_from_value_bytes(alloc: &A, value: &T) -> RcValuePointer + where + A: Allocator, + T: ?Sized, + { + // SAFETY: `value` provides both layout and data source to copy from. + let ptr = unsafe { + allocate_from_bytes( + alloc, + RcLayout::from_value::(value), + NonNull::from_ref(value).cast(), + ) + }; + + ptr.unerase_with(ptr::metadata(value)) + } + + #[cfg(not(no_global_oom_handling))] + pub(super) fn allocate_from_trusted_len(alloc: &A, iter: I) -> RcValuePointer<[I::Item]> + where + A: Allocator, + I: TrustedLen, + { + /// Returns a drop guard that calls the destructors of a slice of elements on drop. + /// + /// # Safety + /// + /// - `head..tail` must describe a valid consecutive slice of `T` values when the destructor + /// of the returned guard is called. + /// - After calling the returned function, the corresponding values should not be accessed + /// any more. + unsafe fn drop_range_on_drop( + head: NonNull, + tail: NonNull, + ) -> DropGuard<(NonNull, NonNull), impl FnOnce((NonNull, NonNull))> { + // SAFETY: + DropGuard::new((head, tail), |(head, tail)| unsafe { + let length = tail.offset_from_unsigned(head); + + NonNull::<[T]>::slice_from_raw_parts(head, length).drop_in_place(); + }) + } + + let (length, Some(high)) = iter.size_hint() else { + // TrustedLen contract guarantees that `upper_bound == None` implies an iterator + // length exceeding `usize::MAX`. + // The default implementation would collect into a vec which would panic. + // Thus we panic here immediately without invoking `Vec` code. + panic!("capacity overflow"); + }; + + debug_assert_eq!( + length, + high, + "TrustedLen iterator's size hint is not exact: {:?}", + (length, high) + ); + + let rc_layout = RcLayout::new_array::(length); + + let ptr = allocate_with(alloc, rc_layout, |ptr| { + let ptr = ptr.unerase::(); + + // SAFETY: The algorithm will ensure the range's validity. + let mut guard = unsafe { drop_range_on_drop(ptr.ptr, ptr.ptr) }; + + // SAFETY: `iter` is `TrustedLen`, we can assume we will write correct number of + // elements to the buffer. + iter.for_each(|value| unsafe { + guard.1.write(value); + guard.1 = guard.1.add(1); + }); + + DropGuard::dismiss(guard); + }); + + ptr.unerase_with(length) + } + + /// Allocates a reference-counted memory chunk to store values produced by `iter` using the + /// `Global` allocator. + #[cfg(not(no_global_oom_handling))] + pub(super) fn allocate_from_iter(iter: I) -> RcValuePointer<[I::Item]> + where + I: Iterator, + { + trait SpecAllocate: Iterator { + fn spec_allocate(self) -> RcValuePointer<[Self::Item]>; + } + + impl SpecAllocate for I + where + I: Iterator, + { + default fn spec_allocate(self) -> RcValuePointer<[Self::Item]> { + Header::::allocate_from_vec(self.collect::>()).0 + } + } + + impl SpecAllocate for I + where + I: TrustedLen, + { + fn spec_allocate(self) -> RcValuePointer<[Self::Item]> { + Header::::allocate_from_trusted_len(&Global, self) + } + } + + iter.spec_allocate::() + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn allocate_from_box(b: Box) -> (RcValuePointer, A) + where + T: ?Sized, + A: Allocator, + { + let ptr = Self::allocate_from_value_bytes(Box::allocator(&b), &*b); + let (box_ptr, alloc) = Box::into_raw_with_allocator(b); + + // SAFETY: Ownership of `T` is transferred into the newly allocated `ptr`, we free the old + // `Box` memory. + unsafe { drop(Box::, &A>::from_raw_in(box_ptr as _, &alloc)) }; + + (ptr, alloc) + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn allocate_from_vec(vec: Vec) -> (RcValuePointer<[T]>, A) + where + A: Allocator, + { + let ptr = Self::allocate_from_value_bytes(vec.allocator(), vec.as_slice()); + let (vec_ptr, _, capacity, alloc) = vec.into_parts_with_allocator(); + + // SAFETY: Ownership of `[T]` is transferred into the newly allocated `ptr`, we free the old + // `Vec` memory. + unsafe { drop(Vec::from_parts_in(vec_ptr, 0, capacity, &alloc)) }; + + (ptr, alloc) + } + + /// # Safety + /// + /// - `ptr` must point to a valid reference counted allocation with header of type `Self`. + /// - Caller must ensure there is no further access to the allocation after calling `deallocate`. + #[inline] + pub(super) unsafe fn deallocate(alloc: &A, ptr: RcValuePointer) + where + A: Allocator, + T: ?Sized, + { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { + deallocate(alloc, ptr.erase(), RcLayout::from_value_ptr_unchecked::(ptr.ptr)); + } + } + + /// # Safety + /// + /// `ptr` must point to a valid reference counted allocation containing an initialized header of + /// type `Self`. + #[inline] + pub(super) unsafe fn get<'a>(ptr: ErasedRcValuePointer) -> &'a Self { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { ptr.ptr.cast().sub(1).as_ref() } + } + + /// # Safety + /// + /// `ptr` must point to a valid reference counted allocation that can store a header of type + /// `Self`. + #[inline] + pub(super) unsafe fn write_to(self, ptr: ErasedRcValuePointer) { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { ptr.ptr.cast().sub(1).write(self) }; + } +} + +/// Tries to allocate uninitialized memory for a reference-counted allocation using allocator +/// `alloc` and layout `rc_layout`. Returns a pointer to the value location. +#[inline] +fn try_allocate_uninit( + alloc: &A, + rc_layout: RcLayout, +) -> Result +where + A: Allocator, +{ + let allocation_ptr = alloc.allocate(rc_layout.layout())?; + + // SAFETY: `allocation_ptr` is allocated with `rc_layout`, so we can safely acquire a pointer to + // the value. + Ok(ErasedRcValuePointer::new(unsafe { + allocation_ptr.cast().byte_add(rc_layout.value_offset()) + })) +} + +/// Tries to allocate zeroed memory for a reference-counted allocation using allocator +/// `alloc` and layout `rc_layout`. Returns a pointer to the value location. +#[inline] +fn try_allocate_zeroed( + alloc: &A, + rc_layout: RcLayout, +) -> Result +where + A: Allocator, +{ + let allocation_ptr = alloc.allocate_zeroed(rc_layout.layout())?; + + // SAFETY: `allocation_ptr` is allocated with `rc_layout`, so we can safely acquire a pointer to + // the value. + Ok(ErasedRcValuePointer::new(unsafe { + allocation_ptr.cast().byte_add(rc_layout.value_offset()) + })) +} + +/// Tries to allocate a reference-counted memory chunk for storing a value according to `rc_layout`, +/// then initializes the value with `f`. If `f` panics, the allocated memory will be deallocated. +#[inline] +fn try_allocate_with( + alloc: &A, + rc_layout: RcLayout, + f: F, +) -> Result +where + A: Allocator, + F: FnOnce(ErasedRcValuePointer), +{ + let ptr = try_allocate_uninit(alloc, rc_layout)?; + + // SAFETY: `ptr` is allocated with `alloc` and `rc_layout`. + let guard = unsafe { deallocate_on_drop(alloc, ptr, rc_layout) }; + + f(ptr); + + // `f` did not panic, dismiss the drop guard. + DropGuard::dismiss(guard); + + Ok(ptr) +} + +/// Allocates uninitialized memory for a reference-counted allocation using allocator `alloc` and +/// layout `rc_layout`. Returns a pointer to the value location. If the allocation fails, panic will +/// be triggered by calling `alloc::handle_alloc_error`. +#[cfg(not(no_global_oom_handling))] +#[inline] +fn allocate_uninit(alloc: &A, rc_layout: RcLayout) -> ErasedRcValuePointer +where + A: Allocator, +{ + match try_allocate_uninit(alloc, rc_layout) { + Ok(result) => result, + Err(_) => alloc::handle_alloc_error(rc_layout.layout()), + } +} + +/// Allocates zeroed memory for a reference-counted allocation using allocator `alloc` and +/// layout `rc_layout`. Returns a pointer to the value location. If the allocation fails, panic will +/// be triggered by calling `alloc::handle_alloc_error`. +#[cfg(not(no_global_oom_handling))] +#[inline] +fn allocate_zeroed(alloc: &A, rc_layout: RcLayout) -> ErasedRcValuePointer +where + A: Allocator, +{ + match try_allocate_zeroed(alloc, rc_layout) { + Ok(result) => result, + Err(_) => alloc::handle_alloc_error(rc_layout.layout()), + } +} + +/// Allocates a reference-counted memory chunk for storing a value according to `rc_layout`, then +/// initializes the value with `f`. If `f` panics, the allocated memory will be deallocated. +#[cfg(not(no_global_oom_handling))] +#[inline] +fn allocate_with(alloc: &A, rc_layout: RcLayout, f: F) -> ErasedRcValuePointer +where + A: Allocator, + F: FnOnce(ErasedRcValuePointer), +{ + let value_ptr = allocate_uninit(alloc, rc_layout); + + // SAFETY: `value_ptr` is allocated with `alloc` and `rc_layout`. + let guard = unsafe { deallocate_on_drop(alloc, value_ptr, rc_layout) }; + + f(value_ptr); + + // `f` did not panic, dismiss the drop guard. + DropGuard::dismiss(guard); + + value_ptr +} + +/// Allocates a reference-counted memory chunk for storing a value according to `rc_layout`, then +/// initializes the value by copying from `source`. +/// +/// # Safety +/// +/// `source` must point to a location that has enough data to copy from. +#[cfg(not(no_global_oom_handling))] +#[inline] +unsafe fn allocate_from_bytes( + alloc: &A, + rc_layout: RcLayout, + source: NonNull<()>, +) -> ErasedRcValuePointer +where + A: Allocator, +{ + let ptr = allocate_uninit(alloc, rc_layout); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it, and caller guarantees the validity of `ptr`. + unsafe { + ptr::copy_nonoverlapping::( + source.cast().as_ptr(), + ptr.ptr.cast().as_ptr(), + rc_layout.value_size(), + ); + } + + ptr +} + +/// Deallocates a reference-counted allocation with a value object pointed to by `value_ptr`. +/// +/// # Safety +/// +/// - `value_ptr` points to a valid reference-counted allocation that was allocated using +/// `rc_layout`. +#[inline] +unsafe fn deallocate(alloc: &A, value_ptr: ErasedRcValuePointer, rc_layout: RcLayout) +where + A: Allocator, +{ + let value_offset = rc_layout.value_offset(); + + // SAFETY: Caller guarantees the validity of `value_ptr`. + unsafe { + let allocation_ptr = value_ptr.ptr.byte_sub(value_offset); + + alloc.deallocate(allocation_ptr.cast(), rc_layout.layout()); + } +} + +/// # Safety +/// +/// - `value_ptr` points to a valid value location within a reference-counted allocation that +/// can be described with `rc_layout` and can be deallocated with `alloc`. +/// - No access to the allocation can happen if the destructor of the returned guard gets +/// called. +#[inline] +unsafe fn deallocate_on_drop<'a, A>( + alloc: &'a A, + value_ptr: ErasedRcValuePointer, + rc_layout: RcLayout, +) -> DropGuard<(), impl FnOnce(())> +where + A: Allocator, +{ + // SAFETY: Caller guarantees the validity of all arguments. + DropGuard::new((), move |()| unsafe { deallocate::(alloc, value_ptr, rc_layout) }) +} diff --git a/library/alloc/src/rcs/common/rc_layout.rs b/library/alloc/src/rcs/common/rc_layout.rs new file mode 100644 index 0000000000000..198be63c36562 --- /dev/null +++ b/library/alloc/src/rcs/common/rc_layout.rs @@ -0,0 +1,249 @@ +use core::alloc::{Layout, LayoutError}; +use core::mem::SizedTypeProperties; +use core::ptr::NonNull; + +/// A `Layout` that describes a reference-counted allocation containing a header of type `H`. +pub(super) struct RcLayout { + layout: Layout, + value_offset: usize, +} + +impl RcLayout { + /// Tries to create an `RcLayout` to store value. Returns `Err` if the value is too big to store + /// in a reference-counted allocation. + #[inline] + pub(super) fn try_from_value(value: &T) -> Result + where + T: ?Sized, + { + /// A helper trait for computing `RcLayout` to store a `Self` object. If `Self` is `Sized`, + /// the `RcLayout` value is computed at compile time. + trait SpecTryRcLayout { + fn spec_try_rc_layout(&self) -> Result; + } + + impl SpecTryRcLayout for T + where + T: ?Sized, + { + #[inline] + default fn spec_try_rc_layout(&self) -> Result { + RcLayout::try_from_value_layout::(Layout::for_value(self)) + } + } + + impl SpecTryRcLayout for T { + #[inline] + fn spec_try_rc_layout(&self) -> Result { + // Do we fail the compilation if we know the calculation overflows at compile time? + const { RcLayout::try_from_value_layout::(T::LAYOUT) } + } + } + + T::spec_try_rc_layout::(value) + } + + /// Tries to create an `RcLayout` to store a value with layout `value_layout`. Returns `Err` if + /// `value_layout` is too big to store in a reference-counted allocation. + #[inline] + pub(super) const fn try_from_value_layout( + value_layout: Layout, + ) -> Result { + match H::LAYOUT.extend(value_layout) { + Ok((layout, value_offset)) => Ok(Self { layout, value_offset }), + Err(error) => Err(error), + } + } + + pub(super) const fn new() -> Self { + const { + match Self::try_from_value_layout::(T::LAYOUT) { + Ok(rc_layout) => rc_layout, + Err(_) => panic!("value is too big to store in a reference-counted allocation"), + } + } + } + + /// Creates an `RcLayout` for storing `value` that is pointed to by `value_ptr`, assuming the + /// value is small enough to fit inside a reference-counted allocation. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn from_value(value: &T) -> Self + where + T: ?Sized, + { + // SAFETY: Pointers derived from references is guaranteed to be valid. + unsafe { Self::from_value_ptr::(NonNull::from_ref(value)) } + } + + /// Creates an `RcLayout` for storing `value`, assuming the value is small enough to fit inside + /// a reference-counted allocation. + /// + /// # Safety + /// + /// Caller guarantees `T` we can store a `T` value inside a reference counted allocation with + /// type `H`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) unsafe fn from_value_unchecked(value: &T) -> Self + where + T: ?Sized, + { + // SAFETY: Caller guarantees we are safe to calculate the allocation layout. + unsafe { Self::from_value_ptr_unchecked::(NonNull::from_ref(value)) } + } + + /// Creates an `RcLayout` for storing a value that is pointed to by `value_ptr`. + /// + /// # Safety + /// + /// `value_ptr` must have correct metadata for `T`. + #[cfg(not(no_global_oom_handling))] + pub(super) unsafe fn from_value_ptr(value_ptr: NonNull) -> Self + where + T: ?Sized, + { + /// A helper trait for computing `RcLayout` to store a `Self` object. If `Self` is `Sized`, + /// the `RcLayout` value is computed at compile time. + trait SpecRcLayout { + unsafe fn spec_rc_layout(value_ptr: NonNull) -> RcLayout; + } + + impl SpecRcLayout for T + where + T: ?Sized, + { + #[inline] + default unsafe fn spec_rc_layout(value_ptr: NonNull) -> RcLayout { + // SAFETY: Caller guarantees the validity of `value_ptr`. + RcLayout::from_value_layout::(unsafe { + Layout::for_value_raw(value_ptr.as_ptr()) + }) + } + } + + impl SpecRcLayout for T { + #[inline] + unsafe fn spec_rc_layout(_: NonNull) -> RcLayout { + const { RcLayout::new::() } + } + } + + // SAFETY: Caller guarantees the validity of `value_ptr`. + unsafe { T::spec_rc_layout::(value_ptr) } + } + + /// Creates an `RcLayout` for storing a value that is pointed to by `value_ptr`, assuming the + /// value is small enough to fit inside a reference-counted allocation. + /// + /// # Safety + /// + /// - `value_ptr` must have correct metadata for a `T` object. + /// - It must be known that the memory layout described by `value_ptr` can be used to create an + /// `RcLayout` successfully. + pub(super) unsafe fn from_value_ptr_unchecked(value_ptr: NonNull) -> Self + where + T: ?Sized, + { + /// A helper trait for computing `RcLayout` to store a `Self` object. If `Self` is `Sized`, + /// the `RcLayout` value is computed at compile time. + trait SpecTryRcLayoutUnchecked { + unsafe fn spec_rc_layout_unchecked(value_ptr: NonNull) -> RcLayout; + } + + impl SpecTryRcLayoutUnchecked for T + where + T: ?Sized, + { + #[inline] + default unsafe fn spec_rc_layout_unchecked(value_ptr: NonNull) -> RcLayout { + // SAFETY: Caller guarantees the validity of `value_ptr`. + unsafe { + RcLayout::from_value_layout_unchecked::(Layout::for_value_raw( + value_ptr.as_ptr(), + )) + } + } + } + + impl SpecTryRcLayoutUnchecked for T { + #[inline] + unsafe fn spec_rc_layout_unchecked(_: NonNull) -> RcLayout { + const { RcLayout::new::() } + } + } + + // SAFETY: Caller guarantees the validity of `value_ptr`. + unsafe { T::spec_rc_layout_unchecked::(value_ptr) } + } + + /// Creates an `RcLayout` to store a value with layout `value_layout`. Panics if `value_layout` + /// is too big to store in a reference-counted allocation. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) fn from_value_layout(value_layout: Layout) -> Self { + Self::try_from_value_layout::(value_layout).expect("capacity overflow") + } + + /// Creates an `RcLayout` to store a value with layout `value_layout`. + /// + /// # Safety + /// + /// `RcLayout::try_from_value_layout(value_layout)` must return `Ok`. + #[inline] + pub(super) const unsafe fn from_value_layout_unchecked(value_layout: Layout) -> Self { + // SAFETY: Caller guarantees the layout create will not fail. + unsafe { + let value_align = value_layout.align(); + + // Calculates `H::SIZE.next_multiple_of(value_align)` given `value_align` being a + // power of 2 and the result does not overflow. + let value_align_m1 = value_align.unchecked_sub(1); + let value_offset = H::SIZE.unchecked_add(value_align_m1) & !value_align_m1; + + let size = value_offset.unchecked_add(value_layout.size()); + let align = if H::ALIGN < value_align { value_align } else { H::ALIGN }; + + Self { layout: Layout::from_size_align_unchecked(size, align), value_offset } + } + } + + /// Creates an `RcLayout` to store an array of `length` elements of type `T`. Panics if the + /// array is too big to store in a reference-counted allocation. + #[cfg(not(no_global_oom_handling))] + pub(super) fn new_array(length: usize) -> Self { + T::LAYOUT + .repeat_packed(length) + .and_then(RcLayout::try_from_value_layout::) + .expect("capacity overflow") + } + + /// Returns an `Layout` object that describes the reference-counted allocation. + pub(super) const fn layout(&self) -> Layout { + self.layout + } + + /// Returns the byte offset of the value stored in a reference-counted allocation that is + /// described by `self`. + #[inline] + pub(super) const fn value_offset(&self) -> usize { + self.value_offset + } + + /// Returns the byte size of the value stored in a reference-counted allocation that is + /// described by `self`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub(super) const fn value_size(&self) -> usize { + // SAFETY: `value_offset` is guaranteed to be less than `size()`. + unsafe { self.layout.size().unchecked_sub(self.value_offset) } + } +} + +impl Clone for RcLayout { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for RcLayout {} diff --git a/library/alloc/src/rcs/common/rc_value_pointer.rs b/library/alloc/src/rcs/common/rc_value_pointer.rs new file mode 100644 index 0000000000000..7d7409eea3fc6 --- /dev/null +++ b/library/alloc/src/rcs/common/rc_value_pointer.rs @@ -0,0 +1,106 @@ +use core::marker::Unsize; +use core::num::NonZero; +use core::ops::{CoerceUnsized, DispatchFromDyn}; +use core::ptr::{NonNull, Pointee}; + +/// An non-null pointer that either: +/// +/// - Points to the value storage for `T` inside a referenced counted allocation. +/// - Is a dangling pointer used in `Weak`. +#[repr(transparent)] +pub(in crate::rcs) struct RcValuePointer +where + T: ?Sized, +{ + pub(in crate::rcs) ptr: NonNull, +} + +impl RcValuePointer +where + T: ?Sized, +{ + pub(in crate::rcs) const fn new(ptr: NonNull) -> Self { + Self { ptr } + } + + pub(in crate::rcs) fn as_ptr(self) -> *mut T { + self.ptr.as_ptr() + } + + pub(in crate::rcs) fn cast(self) -> RcValuePointer { + RcValuePointer::new(self.ptr.cast()) + } + + pub(in crate::rcs) fn erase(self) -> ErasedRcValuePointer { + ErasedRcValuePointer::new(self.ptr.cast()) + } + + #[inline] + pub(in crate::rcs) fn is_dangling(self) -> bool { + self.erase().is_dangling() + } +} + +impl RcValuePointer { + pub(in crate::rcs) const DANGLING: Self = ErasedRcValuePointer::DANGLING.unerase(); +} + +impl Clone for RcValuePointer +where + T: ?Sized, +{ + fn clone(&self) -> Self { + *self + } +} + +impl CoerceUnsized> for RcValuePointer +where + T: ?Sized + Unsize, + U: ?Sized, +{ +} + +impl Copy for RcValuePointer where T: ?Sized {} + +impl DispatchFromDyn> for RcValuePointer +where + T: ?Sized + Unsize, + U: ?Sized, +{ +} + +/// An non-null pointer that points to the value storage for an unspecified type inside a referenced +/// counted allocation. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub(in crate::rcs) struct ErasedRcValuePointer { + pub(in crate::rcs) ptr: NonNull<()>, +} + +impl ErasedRcValuePointer { + const DANGLING: Self = Self::new(NonNull::without_provenance(NonZero::::MAX)); + + pub(in crate::rcs) const fn new(ptr: NonNull<()>) -> Self { + Self { ptr } + } + + #[inline] + fn is_dangling(self) -> bool { + self.ptr.addr().get() == usize::MAX + } + + pub(in crate::rcs) const fn unerase(self) -> RcValuePointer { + RcValuePointer::new(self.ptr.cast()) + } + + pub(in crate::rcs) const fn unerase_with( + self, + metadata: ::Metadata, + ) -> RcValuePointer + where + T: ?Sized, + { + RcValuePointer::new(NonNull::from_raw_parts(self.ptr, metadata)) + } +} diff --git a/library/alloc/src/rcs/rc.rs b/library/alloc/src/rcs/rc.rs index 696bd733b992d..2f61a8bc18458 100644 --- a/library/alloc/src/rcs/rc.rs +++ b/library/alloc/src/rcs/rc.rs @@ -243,16 +243,11 @@ use core::any::Any; use core::cell::{Cell, CloneFromCell}; -#[cfg(not(no_global_oom_handling))] -use core::clone::TrivialClone; use core::clone::{CloneToUninit, Share, UseCloned}; use core::cmp::Ordering; use core::hash::{Hash, Hasher}; -#[cfg(not(no_global_oom_handling))] -use core::iter; use core::marker::{PhantomData, Unsize}; -use core::mem::{self, Alignment, ManuallyDrop}; -use core::num::NonZeroUsize; +use core::mem::{self, ManuallyDrop}; use core::ops::{CoerceUnsized, Deref, DerefMut, DerefPure, DispatchFromDyn, LegacyReceiver}; #[cfg(not(no_global_oom_handling))] use core::ops::{Residual, Try}; @@ -261,46 +256,27 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::Pin; use core::pin::PinSafePointer; use core::ptr::{self, NonNull, drop_in_place}; -#[cfg(not(no_global_oom_handling))] -use core::slice::from_raw_parts_mut; use core::{borrow, fmt, hint, intrinsics}; -#[cfg(not(no_global_oom_handling))] -use crate::alloc::handle_alloc_error; use crate::alloc::{ AllocError, Allocator, AllocatorClone, AllocatorNightly, Global, Layout, StaticAllocator, }; use crate::borrow::{Cow, ToOwned}; +#[cfg(not(no_global_oom_handling))] use crate::boxed::Box; +use crate::rcs::common::{ErasedRcValuePointer, Header, RcValuePointer}; #[cfg(not(no_global_oom_handling))] use crate::string::String; #[cfg(not(no_global_oom_handling))] use crate::vec::Vec; -// This is repr(C) to future-proof against possible field-reordering, which -// would interfere with otherwise safe [into|from]_raw() of transmutable -// inner types. -// repr(align(2)) (forcing alignment to at least 2) is required because usize -// has 1-byte alignment on AVR. -#[repr(C, align(2))] -struct RcInner { - strong: Cell, - weak: Cell, - value: T, -} +type RcHeader = Header>; -/// Calculate layout for `RcInner` using the inner value's layout -fn rc_inner_layout_for_value_layout(layout: Layout) -> Layout { - // Calculate layout using the given value layout. - // Previously, layout was calculated on the expression - // `&*(ptr as *const RcInner)`, but this created a misaligned - // reference (see #54908). - Layout::new::>() - .extend(layout) - .unwrap_or_else(|_| panic!("capacity overflow")) - .0 - .pad_to_align() -} +// Debug visualizers relies on the these deterministic field offsets. +const _: () = { + assert!(size_of::() - mem::offset_of!(RcHeader, strong) == size_of::() * 2); + assert!(size_of::() - mem::offset_of!(RcHeader, weak) == size_of::()); +}; /// A single-threaded reference-counting pointer. 'Rc' stands for 'Reference /// Counted'. @@ -329,8 +305,8 @@ pub struct Rc< // as `Box` is fundamental! #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] A: Allocator = Global, > { - ptr: NonNull>, - phantom: PhantomData>, + ptr: RcValuePointer, + phantom: PhantomData, alloc: A, } @@ -360,46 +336,42 @@ impl, U: ?Sized> DispatchFromDyn> for Rc {} #[unstable(feature = "cell_get_cloned", issue = "145329")] unsafe impl CloneFromCell for Rc {} -impl Rc { +impl Rc { + /// # Safety + /// + /// - `ptr` must point to a valid reference counted allocation for header of type `RcHeader` + /// and value of `T`. + /// - The `T` value pointed to by `ptr` must be initialized. + /// - All future accesses to the allocation must be done through the returned value. #[inline] - unsafe fn from_inner(ptr: NonNull>) -> Self { + unsafe fn init_from_parts(ptr: RcValuePointer, alloc: A) -> Self { // SAFETY: Upheld by caller. - unsafe { Self::from_inner_in(ptr, Global) } - } + unsafe { + RcHeader::INIT_RC.write_to(ptr.erase()); - #[inline] - unsafe fn from_ptr(ptr: *mut RcInner) -> Self { - // SAFETY: Upheld by caller. - unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) } + Self::from_inner_in(ptr, alloc) + } } -} -impl Rc { #[inline(always)] - fn inner(&self) -> &RcInner { + fn inner(&self) -> &RcHeader { // SAFETY: While this Rc is alive we're guaranteed // that the inner pointer is valid. - unsafe { self.ptr.as_ref() } + unsafe { RcHeader::get(self.ptr.erase()) } } #[inline] - fn into_inner_with_allocator(this: Self) -> (NonNull>, A) { + fn into_inner_with_allocator(this: Self) -> (RcValuePointer, A) { let this = mem::ManuallyDrop::new(this); // SAFETY: Pulling out the allocator we already own. (this.ptr, unsafe { ptr::read(&this.alloc) }) } #[inline] - unsafe fn from_inner_in(ptr: NonNull>, alloc: A) -> Self { + unsafe fn from_inner_in(ptr: RcValuePointer, alloc: A) -> Self { Self { ptr, phantom: PhantomData, alloc } } - #[inline] - unsafe fn from_ptr_in(ptr: *mut RcInner, alloc: A) -> Self { - // SAFETY: Upheld by caller. - unsafe { Self::from_inner_in(NonNull::new_unchecked(ptr), alloc) } - } - // Non-inlined part of `drop`. #[inline(never)] unsafe fn drop_slow(&mut self) { @@ -412,7 +384,7 @@ impl Rc { // We cannot use `get_mut_unchecked` here, because `self.alloc` is borrowed. // SAFETY: `self.ptr` is *not* borrowed. unsafe { - ptr::drop_in_place(&mut (*self.ptr.as_ptr()).value); + self.ptr.ptr.drop_in_place(); } } } @@ -429,18 +401,9 @@ impl Rc { /// ``` #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] + #[inline] pub fn new(value: T) -> Rc { - // SAFETY: There is an implicit weak pointer owned by all the strong - // pointers, which ensures that the weak destructor never frees - // the allocation while the strong destructor is running, even - // if the weak pointer is stored inside the strong one. - unsafe { - Self::from_inner(Box::into_non_null(Box::new(RcInner { - strong: Cell::new(1), - weak: Cell::new(1), - value, - }))) - } + Self::new_in(value, Global) } /// Constructs a new `Rc` while giving you a `Weak` to the allocation, @@ -523,14 +486,7 @@ impl Rc { #[stable(feature = "new_uninit", since = "1.82.0")] #[must_use] pub fn new_uninit() -> Rc> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::from_ptr(Rc::allocate_for_layout( - Layout::new::(), - |layout| Global.allocate(layout), - <*mut u8>::cast, - )) - } + Self::new_uninit_in(Global) } /// Constructs a new `Rc` with uninitialized contents, with the memory @@ -555,14 +511,7 @@ impl Rc { #[stable(feature = "new_zeroed_alloc", since = "1.92.0")] #[must_use] pub fn new_zeroed() -> Rc> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::from_ptr(Rc::allocate_for_layout( - Layout::new::(), - |layout| Global.allocate_zeroed(layout), - <*mut u8>::cast, - )) - } + Self::new_zeroed_in(Global) } /// Constructs a new `Rc`, returning an error if the allocation fails @@ -578,17 +527,7 @@ impl Rc { /// ``` #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_new(value: T) -> Result, AllocError> { - // SAFETY: There is an implicit weak pointer owned by all the strong - // pointers, which ensures that the weak destructor never frees - // the allocation while the strong destructor is running, even - // if the weak pointer is stored inside the strong one. - unsafe { - Ok(Self::from_inner(Box::into_non_null(Box::try_new(RcInner { - strong: Cell::new(1), - weak: Cell::new(1), - value, - })?))) - } + Self::try_new_in(value, Global) } /// Constructs a new `Rc` with uninitialized contents, returning an error if the allocation fails @@ -612,14 +551,7 @@ impl Rc { /// ``` #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_new_uninit() -> Result>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Rc::from_ptr(Rc::try_allocate_for_layout( - Layout::new::(), - |layout| Global.allocate(layout), - <*mut u8>::cast, - )?)) - } + Self::try_new_uninit_in(Global) } /// Constructs a new `Rc` with uninitialized contents, with the memory @@ -645,14 +577,7 @@ impl Rc { /// [zeroed]: mem::MaybeUninit::zeroed #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_new_zeroed() -> Result>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Rc::from_ptr(Rc::try_allocate_for_layout( - Layout::new::(), - |layout| Global.allocate_zeroed(layout), - <*mut u8>::cast, - )?)) - } + Self::try_new_zeroed_in(Global) } /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then /// `value` will be pinned in memory and unable to be moved. @@ -682,11 +607,14 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_in(value: T, alloc: A) -> Rc { - // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. - // That would make code size bigger. - match Self::try_new_in(value, alloc) { - Ok(m) => m, - Err(_) => handle_alloc_error(Layout::new::>()), + let mut uninit_rc = Self::new_uninit_in(alloc); + + // SAFETY: `uninit_rc` points to a newly allocated reference counted allocation, we have + // exclusive access to the contained value. + unsafe { + Rc::get_mut_unchecked(&mut uninit_rc).write(value); + + uninit_rc.assume_init() } } @@ -716,17 +644,11 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_uninit_in(alloc: A) -> Rc, A> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::from_ptr_in( - Rc::allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate(layout), - <*mut u8>::cast, - ), - alloc, - ) - } + let ptr = RcHeader::allocate_uninit_for_type(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Rc::init_from_parts(ptr, alloc) } } /// Constructs a new `Rc` with uninitialized contents, with the memory @@ -754,17 +676,11 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_zeroed_in(alloc: A) -> Rc, A> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::from_ptr_in( - Rc::allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate_zeroed(layout), - <*mut u8>::cast, - ), - alloc, - ) - } + let ptr = RcHeader::allocate_zeroed_for_type(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Rc::init_from_parts(ptr, alloc) } } /// Constructs a new `Rc` in the given allocator while giving you a `Weak` to the allocation, @@ -802,17 +718,13 @@ impl Rc { where F: FnOnce(&Weak) -> T, { - // Construct the inner in the "uninitialized" state with a single + // Construct a `Weak` in the "uninitialized" state with a single // weak reference. - let (uninit_ptr, alloc) = Box::into_non_null_with_allocator(Box::new_in( - RcInner { - strong: Cell::new(0), - weak: Cell::new(1), - value: mem::MaybeUninit::::uninit(), - }, - alloc, - )); - let init_ptr: NonNull> = uninit_ptr.cast(); + + let init_ptr = RcHeader::allocate_uninit_for_type(&alloc); + + // SAFETY: We have exclusive access to `init_ptr`. + unsafe { RcHeader::INIT_UNIQUE_RC.write_to(init_ptr.erase()) }; let weak = Weak { ptr: init_ptr, alloc }; @@ -826,12 +738,13 @@ impl Rc { // ignore-tidy-undocumented-unsafe unsafe { - let inner = init_ptr.as_ptr(); - ptr::write(&raw mut (*inner).value, data); + init_ptr.ptr.write(data); + + let strong = RcHeader::get(init_ptr.erase()).strong_ref(); + let prev_value = strong.get(); - let prev_value = (*inner).strong.get(); debug_assert_eq!(prev_value, 0, "No prior strong references should exist"); - (*inner).strong.set(1); + strong.set(1); // Strong references should collectively own a shared weak reference, // so don't run the destructor for our old weak reference. @@ -859,16 +772,13 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_in(value: T, alloc: A) -> Result { - // There is an implicit weak pointer owned by all the strong - // pointers, which ensures that the weak destructor never frees - // the allocation while the strong destructor is running, even - // if the weak pointer is stored inside the strong one. - let (ptr, alloc) = Box::into_non_null_with_allocator(Box::try_new_in( - RcInner { strong: Cell::new(1), weak: Cell::new(1), value }, - alloc, - )?); - // SAFETY: Pointer is valid. - Ok(unsafe { Self::from_inner_in(ptr, alloc) }) + // SAFETY: `uninit_rc` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + Self::try_new_uninit_in(alloc).map(|mut uninit_rc| unsafe { + Rc::get_mut_unchecked(&mut uninit_rc).write(value); + + uninit_rc.assume_init() + }) } /// Constructs a new `Rc` with uninitialized contents, in the provided allocator, returning an @@ -898,17 +808,11 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_uninit_in(alloc: A) -> Result, A>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Rc::from_ptr_in( - Rc::try_allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate(layout), - <*mut u8>::cast, - )?, - alloc, - )) - } + RcHeader::try_allocate_uninit_for_type(&alloc).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Rc::init_from_parts(ptr, alloc) } + }) } /// Constructs a new `Rc` with uninitialized contents, with the memory @@ -937,17 +841,11 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_zeroed_in(alloc: A) -> Result, A>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Rc::from_ptr_in( - Rc::try_allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate_zeroed(layout), - <*mut u8>::cast, - )?, - alloc, - )) - } + RcHeader::try_allocate_zeroed_for_type(&alloc).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Rc::init_from_parts(ptr, alloc) } + }) } /// Constructs a new `Pin>` in the provided allocator. If `T` does not implement `Unpin`, then @@ -1164,8 +1062,7 @@ impl Rc<[T]> { #[stable(feature = "new_uninit", since = "1.82.0")] #[must_use] pub fn new_uninit_slice(len: usize) -> Rc<[mem::MaybeUninit]> { - // ignore-tidy-undocumented-unsafe - unsafe { Rc::from_ptr(Rc::allocate_for_slice(len)) } + Self::new_uninit_slice_in(len, Global) } /// Constructs a new reference-counted slice with uninitialized contents, with the memory being @@ -1190,14 +1087,7 @@ impl Rc<[T]> { #[stable(feature = "new_zeroed_alloc", since = "1.92.0")] #[must_use] pub fn new_zeroed_slice(len: usize) -> Rc<[mem::MaybeUninit]> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::from_ptr(Rc::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| Global.allocate_zeroed(layout), - |mem| mem.cast::().cast_slice(len) as *mut RcInner<[mem::MaybeUninit]>, - )) - } + Self::new_zeroed_slice_in(len, Global) } } @@ -1230,8 +1120,11 @@ impl Rc<[T], A> { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_uninit_slice_in(len: usize, alloc: A) -> Rc<[mem::MaybeUninit], A> { - // ignore-tidy-undocumented-unsafe - unsafe { Rc::from_ptr_in(Rc::allocate_for_slice_in(len, &alloc), alloc) } + let ptr = RcHeader::allocate_uninit_slice(&alloc, len); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Rc::init_from_parts(ptr, alloc) } } /// Constructs a new reference-counted slice with uninitialized contents, with the memory being @@ -1259,17 +1152,11 @@ impl Rc<[T], A> { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Rc<[mem::MaybeUninit], A> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::from_ptr_in( - Rc::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| alloc.allocate_zeroed(layout), - |mem| mem.cast::().cast_slice(len) as *mut RcInner<[mem::MaybeUninit]>, - ), - alloc, - ) - } + let ptr = RcHeader::allocate_zeroed_slice(&alloc, len); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Rc::init_from_parts(ptr, alloc) } } /// Converts the reference-counted slice into a reference-counted array. @@ -1394,17 +1281,11 @@ impl Rc { #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn clone_from_ref_in(value: &T, alloc: A) -> Rc { - // `in_progress` drops the allocation if we panic before finishing initializing it. - let mut in_progress: UniqueRcUninit = UniqueRcUninit::new(value, alloc); + let ptr = RcHeader::allocate_from_cloning(&alloc, value); - // Initialize with clone of value. - // ignore-tidy-undocumented-unsafe - unsafe { - // Clone. If the clone panics, `in_progress` will be dropped and clean up. - value.clone_to_uninit(in_progress.data_ptr().cast()); - // Cast type of pointer, now that it is initialized. - in_progress.into_rc() - } + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, alloc) } } /// Constructs a new `Rc` with a clone of `value` in the provided allocator, returning an error if allocation fails @@ -1423,19 +1304,11 @@ impl Rc { #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_clone_from_ref_in(value: &T, alloc: A) -> Result, AllocError> { - // `in_progress` drops the allocation if we panic before finishing initializing it. - let mut in_progress: UniqueRcUninit = UniqueRcUninit::try_new(value, alloc)?; - - // Initialize with clone of value. - // ignore-tidy-undocumented-unsafe - let initialized_clone = unsafe { - // Clone. If the clone panics, `in_progress` will be dropped and clean up. - value.clone_to_uninit(in_progress.data_ptr().cast()); - // Cast type of pointer, now that it is initialized. - in_progress.into_rc() - }; - - Ok(initialized_clone) + RcHeader::try_allocate_from_cloning(&alloc, value).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Self::init_from_parts(ptr, alloc) } + }) } } @@ -1472,9 +1345,10 @@ impl Rc<[mem::MaybeUninit], A> { #[stable(feature = "new_uninit", since = "1.82.0")] #[inline] pub unsafe fn assume_init(self) -> Rc<[T], A> { - let (ptr, alloc) = Rc::into_inner_with_allocator(self); - // ignore-tidy-undocumented-unsafe - unsafe { Rc::from_ptr_in(ptr.as_ptr() as _, alloc) } + let (ptr, alloc) = Rc::into_raw_with_allocator(self); + + // SAFETY: Caller guarantees the contained value is initialized. + unsafe { Rc::from_raw_in(ptr as _, alloc) } } } @@ -1682,14 +1556,13 @@ impl Rc { #[must_use] #[unstable(feature = "arc_raw_get_strong", issue = "157021")] pub unsafe fn strong_count_from_raw(ptr: *const T) -> usize { - // SAFETY: Upheld by caller. - let offset = unsafe { data_offset(ptr) }; - // Reverse the offset to find the original RcInner. - // SAFETY: Caller ensures this pointer was to an `Rc` allocation, - // so offsetting must be inbounds. - let rc_ptr = unsafe { ptr.byte_sub(offset) as *mut RcInner }; - // SAFETY: Per the above, an `RcInner` is stored here. - unsafe { (*rc_ptr).strong.get() } + // SAFETY: Caller guarantees `ptr` points to a valid reference counted allocation, we are + // safe to get a reference to the header. + let header = unsafe { + RcHeader::get(ErasedRcValuePointer::new(NonNull::new_unchecked(ptr.cast_mut().cast()))) + }; + + header.strong.get() } } @@ -1752,12 +1625,7 @@ impl Rc { #[stable(feature = "weak_into_raw", since = "1.45.0")] #[rustc_never_returns_null_ptr] pub fn as_ptr(this: &Self) -> *const T { - let ptr: *mut RcInner = NonNull::as_ptr(this.ptr); - - // SAFETY: This cannot go through Deref::deref or Rc::inner because - // this is required to retain raw/mut provenance such that e.g. `get_mut` can - // write through the pointer after the Rc is recovered through `from_raw`. - unsafe { &raw mut (*ptr).value } + this.ptr.as_ptr() } /// Constructs an `Rc` from a raw pointer in the provided allocator. @@ -1833,14 +1701,9 @@ impl Rc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub unsafe fn from_raw_in(ptr: *const T, alloc: A) -> Self { // ignore-tidy-undocumented-unsafe - let offset = unsafe { data_offset(ptr) }; - - // Reverse the offset to find the original RcInner. - // ignore-tidy-undocumented-unsafe - let rc_ptr = unsafe { ptr.byte_sub(offset) as *mut RcInner }; - - // ignore-tidy-undocumented-unsafe - unsafe { Self::from_ptr_in(rc_ptr, alloc) } + unsafe { + Self::from_inner_in(RcValuePointer::new(NonNull::new_unchecked(ptr.cast_mut())), alloc) + } } /// Creates a new [`Weak`] pointer to this allocation. @@ -1863,7 +1726,7 @@ impl Rc { { this.inner().inc_weak(); // Make sure we do not create a dangling Weak - debug_assert!(!is_dangling(this.ptr.as_ptr())); + debug_assert!(!this.ptr.is_dangling()); Weak { ptr: this.ptr, alloc: this.alloc.clone() } } @@ -2141,7 +2004,7 @@ impl Rc { // We are careful to *not* create a reference covering the "count" fields, as // this would conflict with accesses to the reference counts (e.g. by `Weak`). // ignore-tidy-undocumented-unsafe - unsafe { &mut (*this.ptr.as_ptr()).value } + unsafe { this.ptr.ptr.as_mut() } } #[inline] @@ -2230,7 +2093,7 @@ impl Rc { // Can just steal the data, all that's left is Weaks let mut in_progress: UniqueRcUninit = - UniqueRcUninit::new(&**this, this.alloc.clone()); + UniqueRcUninit::new(this, this.alloc.clone()); // ignore-tidy-undocumented-unsafe unsafe { // Initialize `in_progress` with move of **this. @@ -2263,7 +2126,7 @@ impl Rc { // reference count is guaranteed to be 1 at this point, and we required // the `Rc` itself to be `mut`, so we're returning the only possible // reference to the allocation. - unsafe { &mut this.ptr.as_mut().value } + unsafe { this.ptr.ptr.as_mut() } } } @@ -2372,224 +2235,14 @@ impl Rc { } } -impl Rc { - /// Allocates an `RcInner` with sufficient space for - /// a possibly-unsized inner value where the value has the layout provided. - /// - /// The function `mem_to_rc_inner` is called with the data pointer - /// and must return back a (potentially fat)-pointer for the `RcInner`. - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_layout( - value_layout: Layout, - allocate: impl FnOnce(Layout) -> Result, AllocError>, - mem_to_rc_inner: impl FnOnce(*mut u8) -> *mut RcInner, - ) -> *mut RcInner { - let layout = rc_inner_layout_for_value_layout(value_layout); - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::try_allocate_for_layout(value_layout, allocate, mem_to_rc_inner) - .unwrap_or_else(|_| handle_alloc_error(layout)) - } - } - - /// Allocates an `RcInner` with sufficient space for - /// a possibly-unsized inner value where the value has the layout provided, - /// returning an error if allocation fails. - /// - /// The function `mem_to_rc_inner` is called with the data pointer - /// and must return back a (potentially fat)-pointer for the `RcInner`. - #[inline] - unsafe fn try_allocate_for_layout( - value_layout: Layout, - allocate: impl FnOnce(Layout) -> Result, AllocError>, - mem_to_rc_inner: impl FnOnce(*mut u8) -> *mut RcInner, - ) -> Result<*mut RcInner, AllocError> { - let layout = rc_inner_layout_for_value_layout(value_layout); - - // Allocate for the layout. - let ptr = allocate(layout)?; - - // Initialize the RcInner - let inner = mem_to_rc_inner(ptr.as_non_null_ptr().as_ptr()); - // ignore-tidy-undocumented-unsafe - unsafe { - debug_assert_eq!(Layout::for_value_raw(inner), layout); - - (&raw mut (*inner).strong).write(Cell::new(1)); - (&raw mut (*inner).weak).write(Cell::new(1)); - } - - Ok(inner) - } -} - -impl Rc { - /// Allocates an `RcInner` with sufficient space for an unsized inner value - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_ptr_in(ptr: *const T, alloc: &A) -> *mut RcInner { - // Allocate for the `RcInner` using the given value. - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::::allocate_for_layout( - Layout::for_value_raw(ptr), - |layout| alloc.allocate(layout), - |mem| mem.with_metadata_of(ptr as *const RcInner), - ) - } - } - - #[cfg(not(no_global_oom_handling))] - fn from_box_in(src: Box) -> Rc { - let value_size = size_of_val(&*src); - // ignore-tidy-undocumented-unsafe - unsafe { - let ptr = Self::allocate_for_ptr_in(&*src, Box::allocator(&src)); - - // Copy value as bytes - ptr::copy_nonoverlapping( - (&raw const *src) as *const u8, - (&raw mut (*ptr).value) as *mut u8, - value_size, - ); - - // Free the allocation without dropping its contents - let (bptr, alloc) = Box::into_raw_with_allocator(src); - let src = Box::from_raw_in(bptr as *mut mem::ManuallyDrop, &alloc); - drop(src); - - Self::from_ptr_in(ptr, alloc) - } - } -} - -impl Rc<[T]> { - /// Allocates an `RcInner<[T]>` with the given length. - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_slice(len: usize) -> *mut RcInner<[T]> { - // ignore-tidy-undocumented-unsafe - unsafe { - Self::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| Global.allocate(layout), - |mem| mem.cast::().cast_slice(len) as *mut RcInner<[T]>, - ) - } - } - - /// Copy elements from slice into newly allocated `Rc<[T]>` - /// - /// Unsafe because the caller must either take ownership, bind `T: Copy` or - /// bind `T: TrivialClone`. - #[cfg(not(no_global_oom_handling))] - unsafe fn copy_from_slice(v: &[T]) -> Rc<[T]> { - // ignore-tidy-undocumented-unsafe - unsafe { - let ptr = Self::allocate_for_slice(v.len()); - ptr::copy_nonoverlapping(v.as_ptr(), (&raw mut (*ptr).value) as *mut T, v.len()); - Self::from_ptr(ptr) - } - } - - /// Constructs an `Rc<[T]>` from an iterator known to be of a certain size. - /// - /// Behavior is undefined should the size be wrong. - #[cfg(not(no_global_oom_handling))] - unsafe fn from_iter_exact(iter: impl Iterator, len: usize) -> Rc<[T]> { - // Panic guard while cloning T elements. - // In the event of a panic, elements that have been written - // into the new RcInner will be dropped, then the memory freed. - struct Guard { - mem: NonNull, - elems: *mut T, - layout: Layout, - n_elems: usize, - } - - impl Drop for Guard { - fn drop(&mut self) { - // ignore-tidy-undocumented-unsafe - unsafe { - let slice = from_raw_parts_mut(self.elems, self.n_elems); - ptr::drop_in_place(slice); - - Global.deallocate(self.mem, self.layout); - } - } - } - - // ignore-tidy-undocumented-unsafe - unsafe { - let ptr = Self::allocate_for_slice(len); - - let mem = ptr as *mut _ as *mut u8; - let layout = Layout::for_value_raw(ptr); - - // Pointer to first element - let elems = (&raw mut (*ptr).value) as *mut T; - - let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 }; - - for (i, item) in iter.enumerate() { - ptr::write(elems.add(i), item); - guard.n_elems += 1; - } - - // All clear. Forget the guard so it doesn't free the new RcInner. - mem::forget(guard); - - Self::from_ptr(ptr) - } - } -} - -impl Rc<[T], A> { - /// Allocates an `RcInner<[T]>` with the given length. - #[inline] - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_slice_in(len: usize, alloc: &A) -> *mut RcInner<[T]> { - // ignore-tidy-undocumented-unsafe - unsafe { - Rc::<[T]>::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| alloc.allocate(layout), - |mem| mem.cast::().cast_slice(len) as *mut RcInner<[T]>, - ) - } - } -} - -#[cfg(not(no_global_oom_handling))] -/// Specialization trait used for `From<&[T]>`. -trait RcFromSlice { - fn from_slice(slice: &[T]) -> Self; -} - -#[cfg(not(no_global_oom_handling))] -impl RcFromSlice for Rc<[T]> { - #[inline] - default fn from_slice(v: &[T]) -> Self { - // ignore-tidy-undocumented-unsafe - unsafe { Self::from_iter_exact(v.iter().cloned(), v.len()) } - } -} - -#[cfg(not(no_global_oom_handling))] -impl RcFromSlice for Rc<[T]> { - #[inline] - fn from_slice(v: &[T]) -> Self { - // SAFETY: `T` implements `TrivialClone`, so this is sound and equivalent - // to the above. - unsafe { Rc::copy_from_slice(v) } - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl Deref for Rc { type Target = T; #[inline(always)] fn deref(&self) -> &T { - &self.inner().value + // SAFETY: `self.ptr` always points to a valid `T`. + unsafe { self.ptr.ptr.as_ref() } } } @@ -2703,13 +2356,12 @@ impl Default for Rc { /// ``` #[inline] fn default() -> Self { - // ignore-tidy-undocumented-unsafe - unsafe { - Self::from_inner(Box::into_non_null(Box::write( - Box::new_uninit(), - RcInner { strong: Cell::new(1), weak: Cell::new(1), value: T::default() }, - ))) - } + let alloc = Global; + let ptr = RcHeader::allocate_from_default(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation with value + // initialized. + unsafe { Self::init_from_parts(ptr, alloc) } } } @@ -3051,7 +2703,7 @@ impl From<&[T]> for Rc<[T]> { /// ``` #[inline] fn from(v: &[T]) -> Rc<[T]> { - >::from_slice(v) + Rc::clone_from_ref_in(v, Global) } } @@ -3149,7 +2801,11 @@ impl From> for Rc { /// ``` #[inline] fn from(v: Box) -> Rc { - Rc::from_box_in(v) + let (ptr, alloc) = RcHeader::allocate_from_box(v); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, alloc) } } } @@ -3168,19 +2824,11 @@ impl From> for Rc<[T], A> { /// ``` #[inline] fn from(v: Vec) -> Rc<[T], A> { - // ignore-tidy-undocumented-unsafe - unsafe { - let (vec_ptr, len, cap, alloc) = v.into_raw_parts_with_allocator(); - - let rc_ptr = Self::allocate_for_slice_in(len, &alloc); - ptr::copy_nonoverlapping(vec_ptr, (&raw mut (*rc_ptr).value) as *mut T, len); + let (ptr, alloc) = RcHeader::allocate_from_vec(v); - // Create a `Vec` with length 0, to deallocate the buffer - // without dropping its contents or the allocator - let _ = Vec::from_raw_parts_in(vec_ptr, 0, cap, &alloc); - - Self::from_ptr_in(rc_ptr, alloc) - } + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, alloc) } } } @@ -3287,45 +2935,11 @@ impl FromIterator for Rc<[T]> { /// # assert_eq!(&*evens, &*(0..10).collect::>()); /// ``` fn from_iter>(iter: I) -> Self { - ToRcSlice::to_rc_slice(iter.into_iter()) - } -} + let ptr = RcHeader::allocate_from_iter(iter.into_iter()); -/// Specialization trait used for collecting into `Rc<[T]>`. -#[cfg(not(no_global_oom_handling))] -trait ToRcSlice: Iterator + Sized { - fn to_rc_slice(self) -> Rc<[T]>; -} - -#[cfg(not(no_global_oom_handling))] -impl> ToRcSlice for I { - default fn to_rc_slice(self) -> Rc<[T]> { - self.collect::>().into() - } -} - -#[cfg(not(no_global_oom_handling))] -impl> ToRcSlice for I { - fn to_rc_slice(self) -> Rc<[T]> { - // This is the case for a `TrustedLen` iterator. - let (low, high) = self.size_hint(); - if let Some(high) = high { - debug_assert_eq!( - low, - high, - "TrustedLen iterator's size hint is not exact: {:?}", - (low, high) - ); - - // SAFETY: We need to ensure that the iterator has an exact length and we have. - unsafe { Rc::from_iter_exact(self, low) } - } else { - // TrustedLen contract guarantees that `upper_bound == None` implies an iterator - // length exceeding `usize::MAX`. - // The default implementation would collect into a vec which would panic. - // Thus we panic here immediately without invoking `Vec` code. - panic!("capacity overflow"); - } + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, Global) } } } @@ -3361,8 +2975,8 @@ pub struct Weak< // but it is not necessarily a valid pointer. // `Weak::new` sets this to `usize::MAX` so that it doesn’t need // to allocate space on the heap. That's not a value a real pointer - // will ever have because RcInner has alignment at least 2. - ptr: NonNull>, + // will ever have. + ptr: RcValuePointer, alloc: A, } @@ -3400,7 +3014,7 @@ impl Weak { #[rustc_const_stable(feature = "const_weak_new", since = "1.73.0")] #[must_use] pub const fn new() -> Weak { - Weak { ptr: NonNull::without_provenance(NonZeroUsize::MAX), alloc: Global } + Weak { ptr: RcValuePointer::DANGLING, alloc: Global } } } @@ -3422,21 +3036,10 @@ impl Weak { #[inline] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn new_in(alloc: A) -> Weak { - Weak { ptr: NonNull::without_provenance(NonZeroUsize::MAX), alloc } + Weak { ptr: RcValuePointer::DANGLING, alloc } } } -pub(crate) fn is_dangling(ptr: *const T) -> bool { - (ptr.cast::<()>()).addr() == usize::MAX -} - -/// Helper type to allow accessing the reference counts without -/// making any assertions about the data field. -struct WeakInner<'a> { - weak: &'a Cell, - strong: &'a Cell, -} - impl Weak { /// Converts a raw pointer previously created by [`into_raw`] back into `Weak`. /// @@ -3557,18 +3160,7 @@ impl Weak { #[must_use] #[stable(feature = "rc_as_ptr", since = "1.45.0")] pub fn as_ptr(&self) -> *const T { - let ptr: *mut RcInner = NonNull::as_ptr(self.ptr); - - if is_dangling(ptr) { - // If the pointer is dangling, we return the sentinel directly. This cannot be - // a valid payload address, as the payload is at least as aligned as RcInner (usize). - ptr as *const T - } else { - // SAFETY: if is_dangling returns false, then the pointer is dereferenceable. - // The payload may be dropped at this point, and we have to maintain provenance, - // so use raw pointer manipulation. - unsafe { &raw mut (*ptr).value } - } + self.ptr.as_ptr() } /// Consumes the `Weak`, returning the wrapped pointer and allocator. @@ -3656,22 +3248,8 @@ impl Weak { #[inline] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub unsafe fn from_raw_in(ptr: *const T, alloc: A) -> Self { - // See Weak::as_ptr for context on how the input pointer is derived. - - let ptr = if is_dangling(ptr) { - // This is a dangling Weak. - ptr as *mut RcInner - } else { - // Otherwise, we're guaranteed the pointer came from a nondangling Weak. - // SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T. - let offset = unsafe { data_offset(ptr) }; - // Thus, we reverse the offset to get the whole RcInner. - // SAFETY: the pointer originated from a Weak, so this offset is safe. - unsafe { ptr.byte_sub(offset) as *mut RcInner } - }; - - // SAFETY: we now have recovered the original Weak pointer, so can create the Weak. - Weak { ptr: unsafe { NonNull::new_unchecked(ptr) }, alloc } + // SAFETY: Caller guarantees the validity of `ptr`. + Weak { ptr: RcValuePointer::new(unsafe { NonNull::new_unchecked(ptr.cast_mut()) }), alloc } } /// Attempts to upgrade the `Weak` pointer to an [`Rc`], delaying @@ -3749,21 +3327,18 @@ impl Weak { } } - /// Returns `None` when the pointer is dangling and there is no allocated `RcInner`, + /// Returns `None` when the pointer is dangling and there is no allocated value, /// (i.e., when this `Weak` was created by `Weak::new`). #[inline] - fn inner(&self) -> Option> { - if is_dangling(self.ptr.as_ptr()) { + fn inner(&self) -> Option<&RcHeader> { + if self.ptr.is_dangling() { None } else { // We are careful to *not* create a reference covering the "data" field, as // the field may be mutated concurrently (for example, if the last `Rc` // is dropped, the data field will be dropped in-place). // ignore-tidy-undocumented-unsafe - Some(unsafe { - let ptr = self.ptr.as_ptr(); - WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak } - }) + Some(unsafe { RcHeader::get(self.ptr.erase()) }) } } @@ -3848,9 +3423,7 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Weak { // the strong pointers have disappeared. if inner.weak() == 0 { // ignore-tidy-undocumented-unsafe - unsafe { - self.alloc.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr())); - } + unsafe { RcHeader::deallocate(&self.alloc, self.ptr) }; } } } @@ -3916,10 +3489,22 @@ impl Default for Weak { // This should have negligible overhead since you don't actually need to // clone these much in Rust thanks to ownership and move-semantics. -#[doc(hidden)] -trait RcInnerPtr { - fn weak_ref(&self) -> &Cell; - fn strong_ref(&self) -> &Cell; +#[expect(clippy::declare_interior_mutable_const, reason = "by design")] +impl RcHeader { + /// Initial header for newly created `Rc` value. The initial weak count of 1 used for tracking + /// all strong counts. + const INIT_RC: Self = Self { weak: Cell::new(1), strong: Cell::new(1) }; + + /// Initial header for newly created `UniqueRc` value. + const INIT_UNIQUE_RC: Self = Self { weak: Cell::new(1), strong: Cell::new(0) }; + + fn weak_ref(&self) -> &Cell { + &self.weak + } + + fn strong_ref(&self) -> &Cell { + &self.strong + } #[inline] fn strong(&self) -> usize { @@ -3988,30 +3573,6 @@ trait RcInnerPtr { } } -impl RcInnerPtr for RcInner { - #[inline(always)] - fn weak_ref(&self) -> &Cell { - &self.weak - } - - #[inline(always)] - fn strong_ref(&self) -> &Cell { - &self.strong - } -} - -impl<'a> RcInnerPtr for WeakInner<'a> { - #[inline(always)] - fn weak_ref(&self) -> &Cell { - self.weak - } - - #[inline(always)] - fn strong_ref(&self) -> &Cell { - self.strong - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl borrow::Borrow for Rc { fn borrow(&self) -> &T { @@ -4029,28 +3590,6 @@ impl AsRef for Rc { #[stable(feature = "pin", since = "1.33.0")] impl Unpin for Rc {} -/// Gets the offset within an `RcInner` for the payload behind a pointer. -/// -/// # Safety -/// -/// The pointer must point to (and have valid metadata for) a previously -/// valid instance of T, but the T is allowed to be dropped. -unsafe fn data_offset(ptr: *const T) -> usize { - // Align the unsized value to the end of the RcInner. - // Because RcInner is repr(C), it will always be the last field in memory. - // SAFETY: since the only unsized types possible are slices, trait objects, - // and extern types, the input safety requirement is currently enough to - // satisfy the requirements of Alignment::of_val_raw; this is an implementation - // detail of the language that must not be relied upon outside of std. - unsafe { data_offset_alignment(Alignment::of_val_raw(ptr)) } -} - -#[inline] -fn data_offset_alignment(alignment: Alignment) -> usize { - let layout = Layout::new::>(); - layout.size() + layout.padding_needed_for(alignment) -} - /// A uniquely owned [`Rc`]. /// /// This represents an `Rc` that is known to be uniquely owned -- that is, have exactly one strong @@ -4092,9 +3631,9 @@ pub struct UniqueRc< T: ?Sized, #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] A: Allocator = Global, > { - ptr: NonNull>, - // Define the ownership of `RcInner` for drop-check - _marker: PhantomData>, + ptr: RcValuePointer, + // Define the ownership of `T` for drop-check + _marker: PhantomData, // Invariance is necessary for soundness: once other `Weak` // references exist, we already have a form of shared mutability! _marker2: PhantomData<*mut T>, @@ -4372,7 +3911,7 @@ impl UniqueRc { } /// Like [`new`](Self::new), but returns an error if the allocation - /// fails, instead of calling [`handle_alloc_error`]. + /// fails, instead of calling [`handle_alloc_error`](crate::alloc::handle_alloc_error). #[unstable(feature = "unique_rc_arc", issue = "112566")] pub fn try_new(value: T) -> Result { Self::try_new_in(value, Global) @@ -4391,35 +3930,33 @@ impl UniqueRc { // #[unstable(feature = "allocator_api", issue = "163177")] #[must_use] pub fn new_in(value: T, alloc: A) -> Self { - let (ptr, alloc) = Box::into_non_null_with_allocator(Box::new_in( - RcInner { - strong: Cell::new(0), - // keep one weak reference so if all the weak pointers that are created are dropped - // the UniqueRc still stays valid. - weak: Cell::new(1), - value, - }, - alloc, - )); + let ptr = RcHeader::allocate_uninit_for_type(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { + RcHeader::INIT_UNIQUE_RC.write_to(ptr.erase()); + ptr.ptr.write(value); + } + Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc } } /// Like [`new_in`](Self::new_in), but returns an error if the allocation - /// fails, instead of calling [`handle_alloc_error`]. + /// fails, instead of calling [`handle_alloc_error`](crate::alloc::handle_alloc_error). #[unstable(feature = "unique_rc_arc", issue = "112566")] // #[unstable(feature = "allocator_api", issue = "163177")] pub fn try_new_in(value: T, alloc: A) -> Result { - let (ptr, alloc) = Box::into_non_null_with_allocator(Box::try_new_in( - RcInner { - strong: Cell::new(0), - // keep one weak reference so if all the weak pointers that are created are dropped - // the UniqueRc still stays valid. - weak: Cell::new(1), - value, - }, - alloc, - )?); - Ok(Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc }) + RcHeader::try_allocate_uninit_for_type(&alloc).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { + RcHeader::INIT_UNIQUE_RC.write_to(ptr.erase()); + ptr.ptr.write(value); + } + + Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc } + }) } /// Consumes the `UniqueRc`, returning its wrapped value and allocator. @@ -4545,16 +4082,9 @@ impl UniqueRc { impl UniqueRc { #[cfg(not(no_global_oom_handling))] unsafe fn from_raw_with_allocator(ptr: *const T, alloc: A) -> Self { - // SAFETY: Upheld by caller - let offset = unsafe { data_offset(ptr) }; - - // Reverse the offset to find the original RcInner. - // SAFETY: As above. - let rc_ptr = unsafe { ptr.byte_sub(offset) as *mut RcInner }; - Self { // SAFETY: Upheld by caller. - ptr: unsafe { NonNull::new_unchecked(rc_ptr) }, + ptr: RcValuePointer::new(unsafe { NonNull::new_unchecked(ptr.cast_mut()) }), _marker: PhantomData, _marker2: PhantomData, alloc, @@ -4576,7 +4106,7 @@ impl UniqueRc { /// references. #[unstable(feature = "unique_rc_arc", issue = "112566")] pub fn into_rc(this: Self) -> Rc { - let mut this = ManuallyDrop::new(this); + let this = ManuallyDrop::new(this); // Move the allocator out. // SAFETY: `this.alloc` will not be accessed again, nor dropped because it is in @@ -4586,7 +4116,7 @@ impl UniqueRc { // SAFETY: This pointer was allocated at creation time so we know it is valid. unsafe { // Convert our weak reference into a strong reference - this.ptr.as_mut().strong.set(1); + this.inner().strong_ref().set(1); Rc::from_inner_in(this.ptr, alloc) } } @@ -4596,30 +4126,27 @@ impl UniqueRc { this.inner().weak() - 1 } - #[cfg(not(no_global_oom_handling))] - fn inner(&self) -> &RcInner { - // SAFETY: while this UniqueRc is alive we're guaranteed that the inner pointer is valid. - unsafe { self.ptr.as_ref() } + fn inner(&self) -> &RcHeader { + // SAFETY: while this UniqueRc is alive we're guaranteed that the pointer is valid. + unsafe { RcHeader::get(self.ptr.erase()) } } fn as_ptr(this: &Self) -> *const T { - let ptr: *mut RcInner = NonNull::as_ptr(this.ptr); - - // SAFETY: This cannot go through Deref::deref or UniqueRc::inner because + // This cannot go through Deref::deref or UniqueRc::inner because // this is required to retain raw/mut provenance such that e.g. `get_mut` can // write through the pointer after the Rc is recovered through `from_raw`. - unsafe { &raw mut (*ptr).value } + this.ptr.as_ptr() } #[inline] - fn into_inner_with_allocator(this: Self) -> (NonNull>, A) { + fn into_inner_with_allocator(this: Self) -> (RcValuePointer, A) { let this = mem::ManuallyDrop::new(this); // SAFETY: Pointer is valid for reads. (this.ptr, unsafe { ptr::read(&this.alloc) }) } #[inline] - unsafe fn from_inner_in(ptr: NonNull>, alloc: A) -> Self { + unsafe fn from_inner_in(ptr: RcValuePointer, alloc: A) -> Self { Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc } } } @@ -4631,11 +4158,8 @@ impl UniqueRc { /// to a [`Rc`] using [`UniqueRc::into_rc`]. #[unstable(feature = "unique_rc_arc", issue = "112566")] pub fn downgrade(this: &Self) -> Weak { - // SAFETY: This pointer was allocated at creation time and we guarantee that we only have - // one strong reference before converting to a regular Rc. - unsafe { - this.ptr.as_ref().inc_weak(); - } + this.inner().inc_weak(); + Weak { ptr: this.ptr, alloc: this.alloc.clone() } } } @@ -4681,7 +4205,7 @@ impl Deref for UniqueRc { fn deref(&self) -> &T { // SAFETY: This pointer was allocated at creation time so we know it is valid. - unsafe { &self.ptr.as_ref().value } + unsafe { self.ptr.ptr.as_ref() } } } @@ -4691,7 +4215,7 @@ impl DerefMut for UniqueRc { // SAFETY: This pointer was allocated at creation time so we know it is valid. We know we // have unique ownership and therefore it's safe to make a mutable reference because // `UniqueRc` owns the only strong reference to itself. - unsafe { &mut (*self.ptr.as_ptr()).value } + unsafe { self.ptr.ptr.as_mut() } } } @@ -4703,64 +4227,49 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for UniqueRc { // destroy the contained object drop_in_place(DerefMut::deref_mut(self)); + let header = self.inner(); + // remove the implicit "strong weak" pointer now that we've destroyed the contents. - self.ptr.as_ref().dec_weak(); + header.dec_weak(); - if self.ptr.as_ref().weak() == 0 { - self.alloc.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr())); + if header.weak() == 0 { + RcHeader::deallocate(&self.alloc, self.ptr); } } } } -/// A unique owning pointer to a [`RcInner`] **that does not imply the contents are initialized,** -/// but will deallocate it (without dropping the value) when dropped. +/// A unique owning pointer to a reference counted allocation **that does not imply the contents are +/// initialized,** but will deallocate it (without dropping the value) when dropped. /// /// This is a helper for [`Rc::make_mut()`] to ensure correct cleanup on panic. /// It is nearly a duplicate of `UniqueRc, A>` except that it allows `T: !Sized`, /// which `MaybeUninit` does not. +#[cfg(not(no_global_oom_handling))] struct UniqueRcUninit { - ptr: NonNull>, - layout_for_value: Layout, - alloc: Option, + ptr: RcValuePointer, + alloc: A, } +#[cfg(not(no_global_oom_handling))] impl UniqueRcUninit { - /// Allocates a RcInner with layout suitable to contain `for_value` or a clone of it. - #[cfg(not(no_global_oom_handling))] - fn new(for_value: &T, alloc: A) -> UniqueRcUninit { - let layout = Layout::for_value(for_value); - // ignore-tidy-undocumented-unsafe - let ptr = unsafe { - Rc::allocate_for_layout( - layout, - |layout_for_rc_inner| alloc.allocate(layout_for_rc_inner), - |mem| mem.with_metadata_of(ptr::from_ref(for_value) as *const RcInner), - ) - }; - Self { ptr: NonNull::new(ptr).unwrap(), layout_for_value: layout, alloc: Some(alloc) } - } + /// Allocates a reference counted allocation with layout suitable to contain `for_value` or a + /// clone of it. + fn new(rc: &Rc, alloc: A) -> UniqueRcUninit { + // SAFETY: `Rc` proves that `T` is small enough to store inside a reference counted + // allocation. + let ptr = unsafe { RcHeader::allocate_uninit_for_value_unchecked(&alloc, Rc::as_ref(rc)) }; - /// Allocates a RcInner with layout suitable to contain `for_value` or a clone of it, - /// returning an error if allocation fails. - fn try_new(for_value: &T, alloc: A) -> Result, AllocError> { - let layout = Layout::for_value(for_value); - // ignore-tidy-undocumented-unsafe - let ptr = unsafe { - Rc::try_allocate_for_layout( - layout, - |layout_for_rc_inner| alloc.allocate(layout_for_rc_inner), - |mem| mem.with_metadata_of(ptr::from_ref(for_value) as *const RcInner), - )? - }; - Ok(Self { ptr: NonNull::new(ptr).unwrap(), layout_for_value: layout, alloc: Some(alloc) }) + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { RcHeader::INIT_RC.write_to(ptr.erase()) }; + + Self { ptr, alloc } } /// Returns the pointer to be written into to initialize the [`Rc`]. fn data_ptr(&mut self) -> *mut T { - let offset = data_offset_alignment(self.layout_for_value.alignment()); - // ignore-tidy-undocumented-unsafe - unsafe { self.ptr.as_ptr().byte_add(offset) as *mut T } + self.ptr.as_ptr() } /// Upgrade this into a normal [`Rc`]. @@ -4769,27 +4278,25 @@ impl UniqueRcUninit { /// /// The data must have been initialized (by writing to [`Self::data_ptr()`]). unsafe fn into_rc(self) -> Rc { - let mut this = ManuallyDrop::new(self); + let this = ManuallyDrop::new(self); let ptr = this.ptr; - let alloc = this.alloc.take().unwrap(); + + // SAFETY: We are safe to take ownership of the allocator. + let alloc = unsafe { ptr::read(&this.alloc) }; // SAFETY: The pointer is valid as per `UniqueRcUninit::new`, and the caller is responsible // for having initialized the data. - unsafe { Rc::from_ptr_in(ptr.as_ptr(), alloc) } + unsafe { Rc::from_inner_in(ptr, alloc) } } } +#[cfg(not(no_global_oom_handling))] impl Drop for UniqueRcUninit { fn drop(&mut self) { // SAFETY: // * new() produced a pointer safe to deallocate. // * We own the pointer unless into_rc() was called, which forgets us. - unsafe { - self.alloc.take().unwrap().deallocate( - self.ptr.cast(), - rc_inner_layout_for_value_layout(self.layout_for_value), - ); - } + unsafe { RcHeader::deallocate(&self.alloc, self.ptr) }; } } diff --git a/src/etc/gdb_providers.py b/src/etc/gdb_providers.py index 9c50d7e472000..0b6458bf20e81 100644 --- a/src/etc/gdb_providers.py +++ b/src/etc/gdb_providers.py @@ -217,20 +217,53 @@ def display_hint(): return "array" +_USIZE_PTR_TYPE = None + + +def _get_usize_ptr_type(): + global _USIZE_PTR_TYPE + + if _USIZE_PTR_TYPE is None: + architecture = gdb.selected_inferior().architecture() + void_ptr_type = architecture.void_type().pointer() + ptr_size = void_ptr_type.sizeof + _USIZE_PTR_TYPE = architecture.integer_type(8 * ptr_size).pointer() + + return _USIZE_PTR_TYPE + + class StdRcProvider(printer_base): def __init__(self, valobj, is_atomic=False): self._valobj = valobj self._is_atomic = is_atomic - self._ptr = unwrap_unique_or_non_null(valobj["ptr"]) - self._value = self._ptr["data" if is_atomic else "value"] - # FIXME(shua): the debuginfo template type should be 'str' not 'u8' - if self._ptr.type.target().name == "alloc::rc::RcInner": - length = self._valobj["ptr"]["pointer"]["length"] - u8_ptr_ty = gdb.Type.pointer(gdb.lookup_type("u8")) - ptr = self._value.address.reinterpret_cast(u8_ptr_ty) - self._value = ptr.lazy_string(encoding="utf-8", length=length) - self._strong = unwrap_scalar_wrappers(self._ptr["strong"]) - self._weak = unwrap_scalar_wrappers(self._ptr["weak"]) - 1 + + if is_atomic: + self._ptr = unwrap_unique_or_non_null(valobj["ptr"]) + self._value = self._ptr["data"] + # FIXME(shua): the debuginfo template type should be 'str' not 'u8' + if self._ptr.type.target().name == "alloc::rc::RcInner": + length = self._valobj["ptr"]["pointer"]["length"] + u8_ptr_ty = gdb.Type.pointer(gdb.lookup_type("u8")) + ptr = self._value.address.reinterpret_cast(u8_ptr_ty) + self._value = ptr.lazy_string(encoding="utf-8", length=length) + self._strong = unwrap_scalar_wrappers(self._ptr["strong"]) + self._weak = unwrap_scalar_wrappers(self._ptr["weak"]) - 1 + else: + self._ptr = unwrap_unique_or_non_null(valobj["ptr"]["ptr"]) + + if self._ptr.type.name == "*const str": + self._value = self._ptr["data_ptr"].lazy_string( + encoding="utf-8", length=self._ptr["length"] + ) + else: + self._value = self._ptr.dereference() + + usize_ptr_type = _get_usize_ptr_type() + weak_ptr = self._ptr.reinterpret_cast(usize_ptr_type) - 1 + strong_ptr = weak_ptr - 1 + + self._strong = strong_ptr.dereference() + self._weak = weak_ptr.dereference() - 1 def to_string(self): if self._is_atomic: diff --git a/src/etc/lldb_providers.py b/src/etc/lldb_providers.py index 5859852fb6be4..c5c28ac7ad2e5 100644 --- a/src/etc/lldb_providers.py +++ b/src/etc/lldb_providers.py @@ -4,6 +4,7 @@ from enum import Flag, auto from typing import TYPE_CHECKING, Dict, Generator, List, Optional +import lldb from lldb import ( SBData, SBError, @@ -1555,14 +1556,43 @@ def StdRcSummaryProvider(valobj: SBValue, _dict: LLDBOpaque) -> str: return f"strong={strong}, weak={weak}" +_USIZE_TYPE = None + + +def _get_usize_type(target): + global _USIZE_TYPE + + if _USIZE_TYPE is None: + ptr_size = target.addr_size + + for basic_type in [ + lldb.eBasicTypeUnsignedLongLong, + lldb.eBasicTypeUnsignedLong, + lldb.eBasicTypeUnsignedShort, + lldb.eBasicTypeUnsignedInt, + lldb.eBasicTypeUnsignedChar, + ]: + candidate_type = target.GetBasicType(basic_type) + + if candidate_type.GetByteSize() == ptr_size: + _USIZE_TYPE = candidate_type + + break + + if _USIZE_TYPE is None: + _USIZE_TYPE = lldb.SBType() + + return _USIZE_TYPE + + class StdRcSyntheticProvider: """Pretty-printer for alloc::rc::Rc and alloc::sync::Arc - - struct Rc { ptr: NonNull>, ... } + struct RcValuePointer { ptr: NonNull, ... } + struct Rc { ptr: RcValuePointer, ... } rust 1.31.1: struct NonNull { pointer: NonZero<*const T> } rust 1.33.0: struct NonNull { pointer: *const T } struct NonZero(T) - struct RcInner { strong: Cell, weak: Cell, value: T } + struct Header { strong: C, weak: C } struct Arc { ptr: NonNull>, ... } struct ArcInner { strong: atomic::Atomic, weak: atomic::Atomic, data: T } @@ -1571,29 +1601,55 @@ class StdRcSyntheticProvider: def __init__(self, valobj: SBValue, _dict: LLDBOpaque, is_atomic: bool = False): self.valobj = valobj - self.ptr = unwrap_unique_or_non_null(self.valobj.GetChildMemberWithName("ptr")) + if is_atomic: + self.ptr = unwrap_unique_or_non_null( + self.valobj.GetChildMemberWithName("ptr") + ) + + self.value = self.ptr.GetChildMemberWithName("data") + + # infallibly gets an unsigned integer type of at least 64 bits. We don't need to worry + # about whether or not `usize` is actually smaller than that since we don't ever display + # the underlying type to the user anyway + usize_type = valobj.GetTarget().GetBasicType(eBasicTypeUnsignedLongLong) - self.value = self.ptr.GetChildMemberWithName("data" if is_atomic else "value") + self.strong = self.ptr.GetChildMemberWithName("strong").Cast(usize_type) + self.weak = self.ptr.GetChildMemberWithName("weak").Cast(usize_type) - # infallibly gets an unsigned integer type of at least 64 bits. We don't need to worry about - # whether or not `usize` is actually smaller than that since we don't ever display the - # underlying type to the user anyway - usize_type = valobj.GetTarget().GetBasicType(eBasicTypeUnsignedLongLong) + # If the usize type isn't valid due to llvm/llvm-project#196812, not even the type's + # fields will populate. Luckily, `RcInner` is `#[repr(C)]`, so we can infallibly find + # the strong and weak values in memory + if not self.strong.IsValid() or not self.weak.IsValid(): + raw_ptr = self.ptr.Cast(usize_type.GetPointerType()) + addr = raw_ptr.GetValueAsAddress() - self.strong = self.ptr.GetChildMemberWithName("strong").Cast(usize_type) - self.weak = self.ptr.GetChildMemberWithName("weak").Cast(usize_type) + self.strong = self.valobj.CreateValueFromAddress( + "strong", addr, usize_type + ) - # If the usize type isn't valid due to llvm/llvm-project#196812, not even the type's fields - # will populate. Luckily, `RcInner` is `#[repr(C)]`, so we can infallibly find the strong - # and weak values in memory - if not self.strong.IsValid() or not self.weak.IsValid(): - raw_ptr = self.ptr.Cast(usize_type.GetPointerType()) - addr = raw_ptr.GetValueAsAddress() - self.strong = self.valobj.CreateValueFromAddress("strong", addr, usize_type) - self.weak = self.valobj.CreateValueFromAddress( - "weak", addr + usize_type.GetByteSize(), usize_type + self.weak = self.valobj.CreateValueFromAddress( + "weak", addr + usize_type.GetByteSize(), usize_type + ) + else: + ptr = ( + self.valobj.GetChildMemberWithName("ptr") + .GetChildMemberWithName("ptr") + .GetChildMemberWithName("pointer") ) + self.value = ptr.deref.Clone("value") + + target = ptr.GetTarget() + weak_address = ptr.GetValueAsUnsigned() - target.addr_size + strong_address = weak_address - target.addr_size + usize_type = _get_usize_type(target) + + self.strong = ptr.CreateValueFromAddress( + "strong", strong_address, usize_type + ) + + self.weak = ptr.CreateValueFromAddress("weak", weak_address, usize_type) + self.value_builder = ValueBuilder(valobj) self.update() diff --git a/src/etc/natvis/liballoc.natvis b/src/etc/natvis/liballoc.natvis index c8e143b49911b..2b6af726ab578 100644 --- a/src/etc/natvis/liballoc.natvis +++ b/src/etc/natvis/liballoc.natvis @@ -73,59 +73,58 @@ --> - {ptr.pointer->value} + {*ptr.ptr.pointer} - ptr.pointer->value - ptr.pointer->strong - ptr.pointer->weak + *ptr.ptr.pointer + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer - 1)->strong + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer - 1)->weak - ptr.pointer.pointer->strong - ptr.pointer.pointer->weak + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.pointer - 1)->strong + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.pointer - 1)->weak - {{ len={ptr.pointer.length} }} + {{ len={ptr.ptr.pointer.length} }} - ptr.pointer.length - ptr.pointer.data_ptr->strong - ptr.pointer.data_ptr->weak + ptr.ptr.pointer.length + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.data_ptr - 1)->strong + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.data_ptr - 1)->weak - ptr.pointer.length - - ($T1*)(((size_t*)ptr.pointer.data_ptr) + 2) + ptr.ptr.pointer.length + ($T1*)ptr.ptr.pointer.data_ptr - {ptr.pointer->value} + {*ptr.ptr.pointer} - ptr.pointer->value - ptr.pointer->strong - ptr.pointer->weak + *ptr.ptr.pointer + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer - 1)->strong + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer - 1)->weak - ptr.pointer.pointer->strong - ptr.pointer.pointer->weak + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.pointer - 1)->strong + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.pointer - 1)->weak - {{ len={ptr.pointer.length} }} + {{ len={ptr.ptr.pointer.length} }} - ptr.pointer.length - ptr.pointer.data_ptr->strong - ptr.pointer.data_ptr->weak + ptr.ptr.pointer.length + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.data_ptr - 1)->strong + ((alloc::rcs::common::Header<core::cell::Cell<usize>> *)ptr.ptr.pointer.data_ptr - 1)->weak - ptr.pointer.length - ($T1*)(((size_t*)ptr.pointer.data_ptr) + 2) + ptr.ptr.pointer.length + ($T1*)ptr.ptr.pointer.data_ptr diff --git a/src/tools/miri/tests/fail/memleak_rc.stderr b/src/tools/miri/tests/fail/memleak_rc.stderr index 26156050ae5d3..8a00371252069 100644 --- a/src/tools/miri/tests/fail/memleak_rc.stderr +++ b/src/tools/miri/tests/fail/memleak_rc.stderr @@ -1,18 +1,23 @@ error: memory leaked: ALLOC (Rust heap, SIZE, ALIGN), allocated here: - --> RUSTLIB/alloc/src/rcs/rc.rs:LL:CC + --> RUSTLIB/alloc/src/rcs/common/mod.rs:LL:CC | -LL | Self::from_inner(Box::into_non_null(Box::new(RcInner { - | _________________________________________________^ -LL | | strong: Cell::new(1), -LL | | weak: Cell::new(1), -LL | | value, -LL | | }))) - | |______________^ +LL | let allocation_ptr = alloc.allocate(rc_layout.layout())?; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: stack backtrace: - 0: std::rc::Rc::>>::new + 0: alloc::rcs::common::try_allocate_uninit:: + at RUSTLIB/alloc/src/rcs/common/mod.rs:LL:CC + 1: alloc::rcs::common::allocate_uninit:: + at RUSTLIB/alloc/src/rcs/common/mod.rs:LL:CC + 2: alloc::rcs::common::Header::>::allocate_uninit_for_type::>>> + at RUSTLIB/alloc/src/rcs/common/mod.rs:LL:CC + 3: std::rc::Rc::>>::new_uninit_in at RUSTLIB/alloc/src/rcs/rc.rs:LL:CC - 1: main + 4: std::rc::Rc::>>::new_in + at RUSTLIB/alloc/src/rcs/rc.rs:LL:CC + 5: std::rc::Rc::>>::new + at RUSTLIB/alloc/src/rcs/rc.rs:LL:CC + 6: main at tests/fail/memleak_rc.rs:LL:CC note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace diff --git a/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs b/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs new file mode 100644 index 0000000000000..9cfb91791a02f --- /dev/null +++ b/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs @@ -0,0 +1,28 @@ +//@ compile-flags: -O -Z merge-functions=disabled + +#![crate_type = "lib"] + +use std::rc::{self, Rc}; + +// Ensures that we can create array of `Weak`s using `memset`. + +#[no_mangle] +pub fn array_of_rc_weak() -> [rc::Weak; 100] { + // CHECK-LABEL: @array_of_rc_weak( + // CHECK-NEXT: start: + // CHECK-NEXT: call void @llvm.memset. + // CHECK-NEXT: ret void + [(); 100].map(|()| rc::Weak::new()) +} + +// Ensures that we convert `&Option>` and `&Option>` to `Option<&T>` without checking +// for `None`. + +#[no_mangle] +pub fn option_rc_as_deref_no_cmp(rc: &Option>) -> Option<&u32> { + // CHECK-LABEL: @option_rc_as_deref_no_cmp(ptr + // CHECK-NEXT: start: + // CHECK-NEXT: %[[RC:.+]] = load ptr, ptr %rc + // CHECK-NEXT: ret ptr %[[RC]] + rc.as_deref() +} diff --git a/tests/codegen-llvm/placement-new.rs b/tests/codegen-llvm/placement-new.rs index 7f7f0033bece3..5022af8f52f19 100644 --- a/tests/codegen-llvm/placement-new.rs +++ b/tests/codegen-llvm/placement-new.rs @@ -22,9 +22,11 @@ pub fn box_default_inplace() -> Box<(String, String)> { #[no_mangle] pub fn rc_default_inplace() -> Rc<(String, String)> { // CHECK-NOT: alloca - // CHECK: [[RC:%.*]] = {{.*}}call {{.*}}__rust_alloc( + // CHECK: [[RC:%.*]] = {{.*}}call {{.*}}__rust_alloc(i[[#BITS:]] // CHECK-NOT: call void @llvm.memcpy - // CHECK: ret ptr [[RC]] + // CHECK: [[DATA:%.*]] = getelementptr inbounds{{( nuw)?}} i8, ptr [[RC]], i[[#BITS]] [[#div(BITS,4)]] + // CHECK-NOT: call void @llvm.memcpy + // CHECK: ret ptr [[DATA]] Rc::default() } diff --git a/tests/debuginfo/strings-and-strs.rs b/tests/debuginfo/strings-and-strs.rs index 56e9f6a5b9773..86f757609933b 100644 --- a/tests/debuginfo/strings-and-strs.rs +++ b/tests/debuginfo/strings-and-strs.rs @@ -20,13 +20,13 @@ //@ gdb-check:$4 = ("Hello", "World") //@ gdb-command:print str_in_rc -//@ gdb-check:$5 = alloc::rcs::rc::Rc<&str, alloc::alloc::Global> {ptr: core::ptr::non_null::NonNull> {pointer: 0x[...]}, phantom: core::marker::PhantomData>, alloc: alloc::alloc::Global} +//@ gdb-check:$5 = alloc::rcs::rc::Rc<&str, alloc::alloc::Global> {ptr: alloc::rcs::common::rc_value_pointer::RcValuePointer<&str> {ptr: core::ptr::non_null::NonNull<&str> {pointer: 0x[...]}}, phantom: core::marker::PhantomData<&str>, alloc: alloc::alloc::Global} //@ gdb-command:print box_str //@ gdb-check:$6 = alloc::boxed::Box [87, 111, 114, 108, 100] //@ gdb-command:print rc_str -//@ gdb-check:$7 = alloc::rcs::rc::Rc {ptr: core::ptr::non_null::NonNull> {pointer: alloc::rcs::rc::RcInner {strong: core::cell::Cell {value: core::cell::UnsafeCell {value: 1}}, weak: core::cell::Cell {value: core::cell::UnsafeCell {value: 1}}, value: 0x[...]}}, phantom: core::marker::PhantomData>, alloc: alloc::alloc::Global} +//@ gdb-check:$7 = alloc::rcs::rc::Rc {ptr: alloc::rcs::common::rc_value_pointer::RcValuePointer {ptr: core::ptr::non_null::NonNull {pointer: *const str [87, 111, 114, 108, 100]}}, phantom: core::marker::PhantomData, alloc: alloc::alloc::Global} // === LLDB TESTS ================================================================================== //@ lldb-command:run From a19b87cb15b208abdc7005bc45796c8025cd9aa0 Mon Sep 17 00:00:00 2001 From: EFanZh Date: Sun, 11 Oct 2026 23:04:30 +0800 Subject: [PATCH 2/4] Try some optimizations --- library/alloc/src/rcs/common/mod.rs | 144 +++++++++---- library/alloc/src/rcs/common/rc_layout.rs | 10 +- .../alloc/src/rcs/common/rc_value_pointer.rs | 4 +- library/alloc/src/rcs/rc.rs | 197 ++++++++++++------ 4 files changed, 233 insertions(+), 122 deletions(-) diff --git a/library/alloc/src/rcs/common/mod.rs b/library/alloc/src/rcs/common/mod.rs index bf403a16e396f..d1b202f3cfc9f 100644 --- a/library/alloc/src/rcs/common/mod.rs +++ b/library/alloc/src/rcs/common/mod.rs @@ -19,12 +19,15 @@ use core::alloc::{AllocError, Allocator}; use core::clone::CloneToUninit; #[cfg(not(no_global_oom_handling))] use core::iter::TrustedLen; +use core::marker::PhantomData; +#[cfg(not(no_global_oom_handling))] use core::mem::DropGuard; #[cfg(not(no_global_oom_handling))] use core::mem::ManuallyDrop; -use core::ptr; +use core::ops::Deref; #[cfg(not(no_global_oom_handling))] use core::ptr::NonNull; +use core::{mem, ptr}; pub(super) use rc_value_pointer::{ErasedRcValuePointer, RcValuePointer}; @@ -49,7 +52,7 @@ pub(super) struct Header { } impl Header { - #[inline] + #[inline(always)] pub(super) fn try_allocate_uninit_for_type( alloc: &A, ) -> Result, AllocError> @@ -59,7 +62,7 @@ impl Header { Ok(try_allocate_uninit(alloc, RcLayout::new::())?.unerase()) } - #[inline] + #[inline(always)] pub(super) fn try_allocate_zeroed_for_type( alloc: &A, ) -> Result, AllocError> @@ -69,6 +72,7 @@ impl Header { Ok(try_allocate_zeroed(alloc, RcLayout::new::())?.unerase()) } + #[inline(always)] pub(super) fn try_allocate_from_cloning( alloc: &A, value: &T, @@ -82,7 +86,7 @@ impl Header { }; // SAFETY: `ptr` is allocated with layout calculated from `value`, we are safe to clone into it. - let ptr = try_allocate_with(alloc, rc_layout, |ptr| unsafe { + let ptr = try_allocate_with(alloc, rc_layout, move |ptr| unsafe { value.clone_to_uninit(ptr.ptr.cast().as_ptr()); })?; @@ -90,7 +94,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] - #[inline] + #[inline(always)] pub(super) fn allocate_uninit_for_type(alloc: &A) -> RcValuePointer where A: Allocator, @@ -103,7 +107,7 @@ impl Header { /// Caller must ensure that `T` is small enough to store inside a reference counted allocation /// with header of type `Self`. #[cfg(not(no_global_oom_handling))] - #[inline] + #[inline(always)] pub(super) unsafe fn allocate_uninit_for_value_unchecked( alloc: &A, value: &T, @@ -118,6 +122,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] + #[inline(always)] pub(super) fn allocate_uninit_slice(alloc: &A, length: usize) -> RcValuePointer<[T]> where A: Allocator, @@ -126,7 +131,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] - #[inline] + #[inline(always)] pub(super) fn allocate_zeroed_for_type(alloc: &A) -> RcValuePointer where A: Allocator, @@ -135,6 +140,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] + #[inline(always)] pub(super) fn allocate_zeroed_slice(alloc: &A, length: usize) -> RcValuePointer<[T]> where A: Allocator, @@ -143,14 +149,14 @@ impl Header { } #[cfg(not(no_global_oom_handling))] - #[inline] + #[inline(always)] pub(super) fn allocate_from_cloning(alloc: &A, value: &T) -> RcValuePointer where A: Allocator, T: CloneToUninit + ?Sized, { // SAFETY: `ptr` is allocated with layout calculated from `value`, we are safe to clone into it. - let ptr = allocate_with(alloc, RcLayout::from_value::(value), |ptr| unsafe { + let ptr = allocate_with(alloc, RcLayout::from_value::(value), move |ptr| unsafe { value.clone_to_uninit(ptr.ptr.cast().as_ptr()) }); @@ -158,6 +164,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] + #[inline(always)] pub(super) fn allocate_from_default(alloc: &A) -> RcValuePointer where A: Allocator, @@ -175,7 +182,7 @@ impl Header { /// but we don't require `T` to be `Copy`, the caller needs to prevent leaks and double-frees /// manually. #[cfg(not(no_global_oom_handling))] - #[inline] + #[inline(always)] pub(super) fn allocate_from_value_bytes(alloc: &A, value: &T) -> RcValuePointer where A: Allocator, @@ -194,6 +201,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] + #[inline(always)] pub(super) fn allocate_from_trusted_len(alloc: &A, iter: I) -> RcValuePointer<[I::Item]> where A: Allocator, @@ -258,6 +266,7 @@ impl Header { /// Allocates a reference-counted memory chunk to store values produced by `iter` using the /// `Global` allocator. #[cfg(not(no_global_oom_handling))] + #[inline(always)] pub(super) fn allocate_from_iter(iter: I) -> RcValuePointer<[I::Item]> where I: Iterator, @@ -288,7 +297,7 @@ impl Header { } #[cfg(not(no_global_oom_handling))] - #[inline] + #[inline(always)] pub(super) fn allocate_from_box(b: Box) -> (RcValuePointer, A) where T: ?Sized, @@ -324,7 +333,7 @@ impl Header { /// /// - `ptr` must point to a valid reference counted allocation with header of type `Self`. /// - Caller must ensure there is no further access to the allocation after calling `deallocate`. - #[inline] + #[inline(always)] pub(super) unsafe fn deallocate(alloc: &A, ptr: RcValuePointer) where A: Allocator, @@ -336,21 +345,11 @@ impl Header { } } - /// # Safety - /// - /// `ptr` must point to a valid reference counted allocation containing an initialized header of - /// type `Self`. - #[inline] - pub(super) unsafe fn get<'a>(ptr: ErasedRcValuePointer) -> &'a Self { - // SAFETY: Caller guarantees the validity of `ptr`. - unsafe { ptr.ptr.cast().sub(1).as_ref() } - } - /// # Safety /// /// `ptr` must point to a valid reference counted allocation that can store a header of type /// `Self`. - #[inline] + #[inline(always)] pub(super) unsafe fn write_to(self, ptr: ErasedRcValuePointer) { // SAFETY: Caller guarantees the validity of `ptr`. unsafe { ptr.ptr.cast().sub(1).write(self) }; @@ -408,14 +407,12 @@ where F: FnOnce(ErasedRcValuePointer), { let ptr = try_allocate_uninit(alloc, rc_layout)?; - - // SAFETY: `ptr` is allocated with `alloc` and `rc_layout`. - let guard = unsafe { deallocate_on_drop(alloc, ptr, rc_layout) }; + let guard = DeallocateOnDrop(alloc, ptr, rc_layout); f(ptr); // `f` did not panic, dismiss the drop guard. - DropGuard::dismiss(guard); + mem::forget(guard); Ok(ptr) } @@ -453,21 +450,19 @@ where /// Allocates a reference-counted memory chunk for storing a value according to `rc_layout`, then /// initializes the value with `f`. If `f` panics, the allocated memory will be deallocated. #[cfg(not(no_global_oom_handling))] -#[inline] +#[inline(always)] fn allocate_with(alloc: &A, rc_layout: RcLayout, f: F) -> ErasedRcValuePointer where A: Allocator, F: FnOnce(ErasedRcValuePointer), { let value_ptr = allocate_uninit(alloc, rc_layout); - - // SAFETY: `value_ptr` is allocated with `alloc` and `rc_layout`. - let guard = unsafe { deallocate_on_drop(alloc, value_ptr, rc_layout) }; + let guard = DeallocateOnDrop(alloc, value_ptr, rc_layout); f(value_ptr); // `f` did not panic, dismiss the drop guard. - DropGuard::dismiss(guard); + mem::forget(guard); value_ptr } @@ -524,21 +519,78 @@ where } } -/// # Safety -/// -/// - `value_ptr` points to a valid value location within a reference-counted allocation that -/// can be described with `rc_layout` and can be deallocated with `alloc`. -/// - No access to the allocation can happen if the destructor of the returned guard gets -/// called. -#[inline] -unsafe fn deallocate_on_drop<'a, A>( - alloc: &'a A, - value_ptr: ErasedRcValuePointer, - rc_layout: RcLayout, -) -> DropGuard<(), impl FnOnce(())> +struct DeallocateOnDrop<'a, A>(&'a A, ErasedRcValuePointer, RcLayout) +where + A: Allocator; + +impl Drop for DeallocateOnDrop<'_, A> where A: Allocator, { - // SAFETY: Caller guarantees the validity of all arguments. - DropGuard::new((), move |()| unsafe { deallocate::(alloc, value_ptr, rc_layout) }) + fn drop(&mut self) { + // SAFETY: All callsites provide valid arguments. + unsafe { deallocate::(self.0, self.1, self.2) } + } } + +/// Provides safe shared access to `Header`. +pub(super) struct HeaderRef<'a, C> { + ptr: ErasedRcValuePointer, + _marker: PhantomData<&'a C>, +} + +impl<'a, C> HeaderRef<'a, C> { + /// # Safety + /// + /// `ptr` must point to a valid reference counted allocation with a header of type `Header`. + pub(super) const unsafe fn new(ptr: ErasedRcValuePointer) -> Self { + Self { ptr, _marker: PhantomData } + } +} + +impl Deref for HeaderRef<'_, C> { + type Target = Header; + + fn deref(&self) -> &Self::Target { + // SAFETY: `self.ptr` is valid by construction. + unsafe { self.ptr.ptr.cast().sub(1).as_ref() } + } +} + +impl Clone for HeaderRef<'_, C> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for HeaderRef<'_, C> {} + +/// Like `Option>`, but `None` is represented with a dangling pointer. +pub(super) struct MaybeHeaderRef<'a, C> { + ptr: ErasedRcValuePointer, + _marker: PhantomData<&'a C>, +} + +impl<'a, C> MaybeHeaderRef<'a, C> { + /// # Safety + /// + /// `ptr` must either be dangling or point to a valid reference counted allocation with a header + /// of type `Header`. + pub(super) unsafe fn new(ptr: ErasedRcValuePointer) -> Self { + Self { ptr, _marker: PhantomData } + } + + #[inline(always)] + pub(super) fn header(self) -> Option> { + // SAFETY: We + if self.ptr.is_dangling() { None } else { Some(unsafe { HeaderRef::new(self.ptr) }) } + } +} + +impl Clone for MaybeHeaderRef<'_, C> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for MaybeHeaderRef<'_, C> {} diff --git a/library/alloc/src/rcs/common/rc_layout.rs b/library/alloc/src/rcs/common/rc_layout.rs index 198be63c36562..566bc20cce850 100644 --- a/library/alloc/src/rcs/common/rc_layout.rs +++ b/library/alloc/src/rcs/common/rc_layout.rs @@ -113,7 +113,7 @@ impl RcLayout { where T: ?Sized, { - #[inline] + #[inline(always)] default unsafe fn spec_rc_layout(value_ptr: NonNull) -> RcLayout { // SAFETY: Caller guarantees the validity of `value_ptr`. RcLayout::from_value_layout::(unsafe { @@ -123,7 +123,7 @@ impl RcLayout { } impl SpecRcLayout for T { - #[inline] + #[inline(always)] unsafe fn spec_rc_layout(_: NonNull) -> RcLayout { const { RcLayout::new::() } } @@ -155,7 +155,7 @@ impl RcLayout { where T: ?Sized, { - #[inline] + #[inline(always)] default unsafe fn spec_rc_layout_unchecked(value_ptr: NonNull) -> RcLayout { // SAFETY: Caller guarantees the validity of `value_ptr`. unsafe { @@ -167,7 +167,7 @@ impl RcLayout { } impl SpecTryRcLayoutUnchecked for T { - #[inline] + #[inline(always)] unsafe fn spec_rc_layout_unchecked(_: NonNull) -> RcLayout { const { RcLayout::new::() } } @@ -190,7 +190,7 @@ impl RcLayout { /// # Safety /// /// `RcLayout::try_from_value_layout(value_layout)` must return `Ok`. - #[inline] + #[inline(always)] pub(super) const unsafe fn from_value_layout_unchecked(value_layout: Layout) -> Self { // SAFETY: Caller guarantees the layout create will not fail. unsafe { diff --git a/library/alloc/src/rcs/common/rc_value_pointer.rs b/library/alloc/src/rcs/common/rc_value_pointer.rs index 7d7409eea3fc6..51f430f41c337 100644 --- a/library/alloc/src/rcs/common/rc_value_pointer.rs +++ b/library/alloc/src/rcs/common/rc_value_pointer.rs @@ -86,8 +86,8 @@ impl ErasedRcValuePointer { } #[inline] - fn is_dangling(self) -> bool { - self.ptr.addr().get() == usize::MAX + pub(super) fn is_dangling(self) -> bool { + self.ptr.as_ptr().addr() == usize::MAX } pub(in crate::rcs) const fn unerase(self) -> RcValuePointer { diff --git a/library/alloc/src/rcs/rc.rs b/library/alloc/src/rcs/rc.rs index 2f61a8bc18458..fa255de228405 100644 --- a/library/alloc/src/rcs/rc.rs +++ b/library/alloc/src/rcs/rc.rs @@ -264,7 +264,7 @@ use crate::alloc::{ use crate::borrow::{Cow, ToOwned}; #[cfg(not(no_global_oom_handling))] use crate::boxed::Box; -use crate::rcs::common::{ErasedRcValuePointer, Header, RcValuePointer}; +use crate::rcs::common::{ErasedRcValuePointer, Header, HeaderRef, MaybeHeaderRef, RcValuePointer}; #[cfg(not(no_global_oom_handling))] use crate::string::String; #[cfg(not(no_global_oom_handling))] @@ -278,6 +278,36 @@ const _: () = { assert!(size_of::() - mem::offset_of!(RcHeader, weak) == size_of::()); }; +type RcHeaderRef<'a> = HeaderRef<'a, Cell>; +type MaybeRcHeaderRef<'a> = MaybeHeaderRef<'a, Cell>; + +/// Writes initial header for newly created `Rc` value. The initial weak count of 1 used for tracking +/// all strong counts. +/// +/// # Safety +/// +/// `ptr` must point to a valid reference counted allocation with mutable access. +#[inline] +unsafe fn write_rc_header(ptr: ErasedRcValuePointer) { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { + RcHeader { strong: Cell::new(1), weak: Cell::new(1) }.write_to(ptr); + } +} + +/// Writes initial header for newly created `UniqueRc` value. +/// +/// # Safety +/// +/// `ptr` must point to a valid reference counted allocation with mutable access. +#[inline] +unsafe fn write_unique_rc_header(ptr: ErasedRcValuePointer) { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { + RcHeader { strong: Cell::new(0), weak: Cell::new(1) }.write_to(ptr); + } +} + /// A single-threaded reference-counting pointer. 'Rc' stands for 'Reference /// Counted'. /// @@ -347,17 +377,24 @@ impl Rc { unsafe fn init_from_parts(ptr: RcValuePointer, alloc: A) -> Self { // SAFETY: Upheld by caller. unsafe { - RcHeader::INIT_RC.write_to(ptr.erase()); + write_rc_header(ptr.erase()); Self::from_inner_in(ptr, alloc) } } #[inline(always)] - fn inner(&self) -> &RcHeader { - // SAFETY: While this Rc is alive we're guaranteed - // that the inner pointer is valid. - unsafe { RcHeader::get(self.ptr.erase()) } + fn inner(&self) -> RcHeaderRef<'_> { + // SAFETY: `self.ptr` is always valid. + unsafe { RcHeaderRef::new(self.ptr.erase()) } + } + + #[inline(always)] + fn non_dangling_ptr(&self) -> RcValuePointer { + // SAFETY: `self.ptr` is always valid. + unsafe { hint::assert_unchecked(!self.ptr.is_dangling()) } + + self.ptr } #[inline] @@ -375,10 +412,16 @@ impl Rc { // Non-inlined part of `drop`. #[inline(never)] unsafe fn drop_slow(&mut self) { + // ignore-tidy-undocumented-unsafe + unsafe { self.drop_slow_inlined() }; + } + + #[inline(always)] + unsafe fn drop_slow_inlined(&mut self) { // Reconstruct the "strong weak" pointer and drop it when this // variable goes out of scope. This ensures that the memory is // deallocated even if the destructor of `T` panics. - let _weak = Weak { ptr: self.ptr, alloc: &self.alloc }; + let _weak = Weak { ptr: self.non_dangling_ptr(), alloc: &self.alloc }; // Destroy the contained object. // We cannot use `get_mut_unchecked` here, because `self.alloc` is borrowed. @@ -401,7 +444,7 @@ impl Rc { /// ``` #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] - #[inline] + #[inline(always)] pub fn new(value: T) -> Rc { Self::new_in(value, Global) } @@ -605,7 +648,7 @@ impl Rc { /// ``` #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] - #[inline] + #[inline(always)] pub fn new_in(value: T, alloc: A) -> Rc { let mut uninit_rc = Self::new_uninit_in(alloc); @@ -642,7 +685,7 @@ impl Rc { /// ``` #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] - #[inline] + #[inline(always)] pub fn new_uninit_in(alloc: A) -> Rc, A> { let ptr = RcHeader::allocate_uninit_for_type(&alloc); @@ -724,7 +767,10 @@ impl Rc { let init_ptr = RcHeader::allocate_uninit_for_type(&alloc); // SAFETY: We have exclusive access to `init_ptr`. - unsafe { RcHeader::INIT_UNIQUE_RC.write_to(init_ptr.erase()) }; + unsafe { write_unique_rc_header(init_ptr.erase()) }; + + // SAFETY: `init_ptr` is guaranteed to be valid. + unsafe { hint::assert_unchecked(!init_ptr.is_dangling()) }; let weak = Weak { ptr: init_ptr, alloc }; @@ -740,7 +786,8 @@ impl Rc { unsafe { init_ptr.ptr.write(data); - let strong = RcHeader::get(init_ptr.erase()).strong_ref(); + let header_ref = RcHeaderRef::new(init_ptr.erase()); + let strong = header_ref.strong_ref(); let prev_value = strong.get(); debug_assert_eq!(prev_value, 0, "No prior strong references should exist"); @@ -891,12 +938,12 @@ impl Rc { // ignore-tidy-undocumented-unsafe let alloc: A = unsafe { ptr::read(&this.alloc) }; // copy the allocator - // Indicate to Weaks that they can't be promoted by decrementing - // the strong count, and then remove the implicit "strong weak" + // Indicate to Weaks that they can't be promoted by setting strong count to 0, and then + // remove the implicit "strong weak" // pointer while also handling drop logic by just crafting a // fake Weak. - this.inner().dec_strong(); - let _weak = Weak { ptr: this.ptr, alloc }; + this.inner().strong_ref().set(0); + let _weak = Weak { ptr: this.non_dangling_ptr(), alloc: &alloc }; Ok(val) } else { Err(this) @@ -1559,7 +1606,9 @@ impl Rc { // SAFETY: Caller guarantees `ptr` points to a valid reference counted allocation, we are // safe to get a reference to the header. let header = unsafe { - RcHeader::get(ErasedRcValuePointer::new(NonNull::new_unchecked(ptr.cast_mut().cast()))) + RcHeaderRef::new(ErasedRcValuePointer::new(NonNull::new_unchecked( + ptr.cast_mut().cast(), + ))) }; header.strong.get() @@ -1726,8 +1775,7 @@ impl Rc { { this.inner().inc_weak(); // Make sure we do not create a dangling Weak - debug_assert!(!this.ptr.is_dangling()); - Weak { ptr: this.ptr, alloc: this.alloc.clone() } + Weak { ptr: this.non_dangling_ptr(), alloc: this.alloc.clone() } } /// Gets the number of [`Weak`] pointers to this allocation. @@ -1901,7 +1949,7 @@ impl Rc { /// ``` /// /// [`get_mut_unchecked`]: Self::get_mut_unchecked - #[inline] + #[inline(always)] #[unstable(feature = "arc_is_unique", issue = "138938")] pub fn is_unique(this: &Self) -> bool { Rc::weak_count(this) == 0 && Rc::strong_count(this) == 1 @@ -2107,7 +2155,7 @@ impl Rc { // This leaves us with 0 strong refs, so the data has // effectively been moved to the new rc. - this.inner().dec_strong(); + this.inner().strong_ref().set(0); // Remove implicit strong-weak ref (no need to craft a fake // Weak here -- we know other Weaks can clean up for us) @@ -2303,7 +2351,11 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Rc { unsafe { self.inner().dec_strong(); if self.inner().strong() == 0 { - self.drop_slow(); + if const { mem::needs_drop::() } { + self.drop_slow(); + } else { + self.drop_slow_inlined(); + } } } } @@ -3288,7 +3340,7 @@ impl Weak { where A: AllocatorClone, { - let inner = self.inner()?; + let inner = self.inner().header()?; if inner.strong() == 0 { None @@ -3307,7 +3359,7 @@ impl Weak { #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn strong_count(&self) -> usize { - if let Some(inner) = self.inner() { inner.strong() } else { 0 } + if let Some(inner) = self.inner().header() { inner.strong() } else { 0 } } /// Gets the number of `Weak` pointers pointing to this allocation. @@ -3316,7 +3368,7 @@ impl Weak { #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn weak_count(&self) -> usize { - if let Some(inner) = self.inner() { + if let Some(inner) = self.inner().header() { if inner.strong() > 0 { inner.weak() - 1 // subtract the implicit weak ptr } else { @@ -3327,19 +3379,11 @@ impl Weak { } } - /// Returns `None` when the pointer is dangling and there is no allocated value, - /// (i.e., when this `Weak` was created by `Weak::new`). - #[inline] - fn inner(&self) -> Option<&RcHeader> { - if self.ptr.is_dangling() { - None - } else { - // We are careful to *not* create a reference covering the "data" field, as - // the field may be mutated concurrently (for example, if the last `Rc` - // is dropped, the data field will be dropped in-place). - // ignore-tidy-undocumented-unsafe - Some(unsafe { RcHeader::get(self.ptr.erase()) }) - } + /// Returns an accessor to the reference counters. + #[inline(always)] + fn inner(&self) -> MaybeRcHeaderRef<'_> { + // SAFETY: `self.ptr` is either valid or danging. + unsafe { MaybeRcHeaderRef::new(self.ptr.erase()) } } /// Returns `true` if the two `Weak`s point to the same allocation similar to [`ptr::eq`], or if @@ -3416,7 +3460,7 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Weak { /// assert!(other_weak_foo.upgrade().is_none()); /// ``` fn drop(&mut self) { - let inner = if let Some(inner) = self.inner() { inner } else { return }; + let inner = if let Some(inner) = self.inner().header() { inner } else { return }; inner.dec_weak(); // the weak count starts at 1, and will only go to zero if all @@ -3443,9 +3487,15 @@ impl Clone for Weak { /// ``` #[inline] fn clone(&self) -> Weak { - if let Some(inner) = self.inner() { - inner.inc_weak() + #[inline] + fn inc_weak(header_ref: MaybeRcHeaderRef<'_>) { + if let Some(inner) = header_ref.header() { + inner.inc_weak() + } } + + inc_weak(self.inner()); + Weak { ptr: self.ptr, alloc: self.alloc.clone() } } } @@ -3489,30 +3539,24 @@ impl Default for Weak { // This should have negligible overhead since you don't actually need to // clone these much in Rust thanks to ownership and move-semantics. -#[expect(clippy::declare_interior_mutable_const, reason = "by design")] -impl RcHeader { - /// Initial header for newly created `Rc` value. The initial weak count of 1 used for tracking - /// all strong counts. - const INIT_RC: Self = Self { weak: Cell::new(1), strong: Cell::new(1) }; - - /// Initial header for newly created `UniqueRc` value. - const INIT_UNIQUE_RC: Self = Self { weak: Cell::new(1), strong: Cell::new(0) }; - +impl<'a> RcHeaderRef<'a> { + #[inline(always)] fn weak_ref(&self) -> &Cell { &self.weak } + #[inline(always)] fn strong_ref(&self) -> &Cell { &self.strong } - #[inline] - fn strong(&self) -> usize { + #[inline(always)] + fn strong(self) -> usize { self.strong_ref().get() } #[inline] - fn inc_strong(&self) { + fn inc_strong(self) { let strong = self.strong(); // We insert an `assume` here to hint LLVM at an otherwise @@ -3534,18 +3578,22 @@ impl RcHeader { } } - #[inline] - fn dec_strong(&self) { - self.strong_ref().set(self.strong() - 1); + #[inline(always)] + fn dec_strong(self) -> bool { + let strong_ref = self.strong_ref(); + + strong_ref.set(strong_ref.get() - 1); + + strong_ref.get() == 0 } - #[inline] - fn weak(&self) -> usize { + #[inline(always)] + fn weak(self) -> usize { self.weak_ref().get() } #[inline] - fn inc_weak(&self) { + fn inc_weak(self) { let weak = self.weak(); // We insert an `assume` here to hint LLVM at an otherwise @@ -3567,8 +3615,8 @@ impl RcHeader { } } - #[inline] - fn dec_weak(&self) { + #[inline(always)] + fn dec_weak(self) { self.weak_ref().set(self.weak() - 1); } } @@ -3935,7 +3983,7 @@ impl UniqueRc { // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive // access to it. unsafe { - RcHeader::INIT_UNIQUE_RC.write_to(ptr.erase()); + write_unique_rc_header(ptr.erase()); ptr.ptr.write(value); } @@ -3951,7 +3999,7 @@ impl UniqueRc { // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive // access to it. unsafe { - RcHeader::INIT_UNIQUE_RC.write_to(ptr.erase()); + write_unique_rc_header(ptr.erase()); ptr.ptr.write(value); } @@ -3964,7 +4012,7 @@ impl UniqueRc { // #[unstable(feature = "allocator_api", issue = "163177")] #[must_use] pub fn unwrap_with_allocator(this: Self) -> (T, A) { - let inner_ptr = this.ptr; + let inner_ptr = this.non_dangling_ptr(); let (data_ptr, alloc) = Self::into_raw_with_allocator(this); // SAFETY: Conceptually moves out of the `UniqueRc`. @@ -4126,9 +4174,18 @@ impl UniqueRc { this.inner().weak() - 1 } - fn inner(&self) -> &RcHeader { - // SAFETY: while this UniqueRc is alive we're guaranteed that the pointer is valid. - unsafe { RcHeader::get(self.ptr.erase()) } + #[inline(always)] + fn inner(&self) -> RcHeaderRef<'_> { + // SAFETY: `self.ptr` is always valid. + unsafe { RcHeaderRef::new(self.ptr.erase()) } + } + + #[inline(always)] + fn non_dangling_ptr(&self) -> RcValuePointer { + // SAFETY: `UniqueRc` always contains a non-dangling value pointer. + unsafe { hint::assert_unchecked(!self.ptr.is_dangling()) } + + self.ptr } fn as_ptr(this: &Self) -> *const T { @@ -4160,7 +4217,7 @@ impl UniqueRc { pub fn downgrade(this: &Self) -> Weak { this.inner().inc_weak(); - Weak { ptr: this.ptr, alloc: this.alloc.clone() } + Weak { ptr: this.non_dangling_ptr(), alloc: this.alloc.clone() } } } @@ -4255,6 +4312,7 @@ struct UniqueRcUninit { impl UniqueRcUninit { /// Allocates a reference counted allocation with layout suitable to contain `for_value` or a /// clone of it. + #[inline(always)] fn new(rc: &Rc, alloc: A) -> UniqueRcUninit { // SAFETY: `Rc` proves that `T` is small enough to store inside a reference counted // allocation. @@ -4262,7 +4320,7 @@ impl UniqueRcUninit { // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive // access to it. - unsafe { RcHeader::INIT_RC.write_to(ptr.erase()) }; + unsafe { write_rc_header(ptr.erase()) }; Self { ptr, alloc } } @@ -4277,6 +4335,7 @@ impl UniqueRcUninit { /// # Safety /// /// The data must have been initialized (by writing to [`Self::data_ptr()`]). + #[inline] unsafe fn into_rc(self) -> Rc { let this = ManuallyDrop::new(self); let ptr = this.ptr; From b5e1615af8a2dcafbd923659b77611b476c0437d Mon Sep 17 00:00:00 2001 From: EFanZh Date: Wed, 7 Oct 2026 21:17:08 +0800 Subject: [PATCH 3/4] Make `Arc::deref` zero-cost --- library/alloc/src/rcs/arc.rs | 1121 +++++------------ src/etc/gdb_providers.py | 36 +- src/etc/lldb_lookup.py | 6 +- src/etc/lldb_providers.py | 63 +- src/etc/natvis/liballoc.natvis | 48 +- tests/codegen-llvm/issues/issue-111603.rs | 8 +- .../lib-optimizations/rc-arc-optimizations.rs | 19 + tests/codegen-llvm/placement-new.rs | 6 +- tests/debuginfo/rc_arc.rs | 2 +- 9 files changed, 402 insertions(+), 907 deletions(-) diff --git a/library/alloc/src/rcs/arc.rs b/library/alloc/src/rcs/arc.rs index 9abfa8f8a862d..018f552219324 100644 --- a/library/alloc/src/rcs/arc.rs +++ b/library/alloc/src/rcs/arc.rs @@ -1,34 +1,29 @@ use core::any::Any; use core::cell::CloneFromCell; -#[cfg(not(no_global_oom_handling))] -use core::clone::TrivialClone; use core::clone::{CloneToUninit, Share, UseCloned}; use core::cmp::Ordering; use core::hash::{Hash, Hasher}; -#[cfg(not(no_global_oom_handling))] -use core::iter; use core::marker::{PhantomData, Unsize}; -use core::mem::{self, Alignment, ManuallyDrop}; -use core::num::NonZeroUsize; +use core::mem::{self, ManuallyDrop, MaybeUninit}; use core::ops::{CoerceUnsized, Deref, DerefMut, DerefPure, DispatchFromDyn, LegacyReceiver}; #[cfg(not(no_global_oom_handling))] use core::ops::{Residual, Try}; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::pin::{Pin, PinSafePointer}; -use core::ptr::{self, NonNull}; #[cfg(not(no_global_oom_handling))] -use core::slice::from_raw_parts_mut; +use core::ptr::Pointee; +use core::ptr::{self, NonNull}; use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use core::sync::atomic::{self, Atomic}; use core::{borrow, fmt, hint, intrinsics}; -#[cfg(not(no_global_oom_handling))] -use crate::alloc::handle_alloc_error; use crate::alloc::{ AllocError, Allocator, AllocatorClone, AllocatorNightly, Global, Layout, StaticAllocator, }; use crate::borrow::{Cow, ToOwned}; +#[cfg(not(no_global_oom_handling))] use crate::boxed::Box; +use crate::rcs::common::{ErasedRcValuePointer, Header, RcValuePointer}; #[cfg(not(no_global_oom_handling))] use crate::string::String; #[cfg(not(no_global_oom_handling))] @@ -45,13 +40,6 @@ use crate::vec::Vec; /// See comment in `Arc::clone`. const MAX_REFCOUNT: usize = (isize::MAX) as usize; -fn is_dangling(ptr: *const T) -> bool -where - T: ?Sized, -{ - ptr.addr() == usize::MAX -} - #[cold] #[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[cfg_attr(panic = "immediate-abort", inline)] @@ -275,8 +263,8 @@ pub struct Arc< // as `Box` is fundamental! #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] A: Allocator = Global, > { - ptr: NonNull>, - phantom: PhantomData>, + ptr: RcValuePointer, + phantom: PhantomData, alloc: A, } @@ -301,35 +289,36 @@ impl, U: ?Sized> DispatchFromDyn> for Arc {} #[unstable(feature = "cell_get_cloned", issue = "145329")] unsafe impl CloneFromCell for Arc {} -impl Arc { - unsafe fn from_inner(ptr: NonNull>) -> Self { - // SAFETY: Upheld by caller. - unsafe { Self::from_inner_in(ptr, Global) } - } - - unsafe fn from_ptr(ptr: *mut ArcInner) -> Self { - // SAFETY: Upheld by caller. - unsafe { Self::from_ptr_in(ptr, Global) } - } -} - impl Arc { #[inline] - fn into_inner_with_allocator(this: Self) -> (NonNull>, A) { + fn into_inner_with_allocator(this: Self) -> (RcValuePointer, A) { let this = mem::ManuallyDrop::new(this); // SAFETY: Pointer is valid for reads. (this.ptr, unsafe { ptr::read(&this.alloc) }) } #[inline] - unsafe fn from_inner_in(ptr: NonNull>, alloc: A) -> Self { + unsafe fn from_inner_in(ptr: RcValuePointer, alloc: A) -> Self { Self { ptr, phantom: PhantomData, alloc } } + /// # Safety + /// + /// - `ptr` must point to a valid reference counted allocation for header of type `ArcHeader` + /// and value of `T`. + /// - The `T` value pointed to by `ptr` must be initialized. + /// - All future accesses to the allocation must be done through the returned value. #[inline] - unsafe fn from_ptr_in(ptr: *mut ArcInner, alloc: A) -> Self { + unsafe fn init_from_parts(ptr: RcValuePointer, alloc: A) -> Self { // SAFETY: Upheld by caller. - unsafe { Self::from_inner_in(NonNull::new_unchecked(ptr), alloc) } + unsafe { + // `ArcHeader::INIT_ARC` is no used here because that generates `memcpy` calls that fails + // `tests/codegen-llvm/placement-new.rs` test. + ArcHeader { weak: Atomic::::new(1), strong: Atomic::::new(1) } + .write_to(ptr.erase()); + + Self::from_inner_in(ptr, alloc) + } } } @@ -363,10 +352,10 @@ pub struct Weak< > { // This is a `NonNull` to allow optimizing the size of this type in enums, // but it is not necessarily a valid pointer. - // `Weak::new` sets this to `usize::MAX` so that it doesn’t need + // `Weak::new` sets this to a dangling pointer so that it doesn’t need // to allocate space on the heap. That's not a value a real pointer - // will ever have because ArcInner has alignment at least 2. - ptr: NonNull>, + // will ever have. + ptr: RcValuePointer, alloc: A, } @@ -391,42 +380,24 @@ impl fmt::Debug for Weak { } } -// This is repr(C) to future-proof against possible field-reordering, which -// would interfere with otherwise safe [into|from]_raw() of transmutable -// inner types. -// Unlike RcInner, repr(align(2)) is not strictly required because atomic types -// have the alignment same as its size, but we use it for consistency and clarity. -#[repr(C, align(2))] -struct ArcInner { - strong: Atomic, - - // the value usize::MAX acts as a sentinel for temporarily "locking" the - // weak count, preventing `Arc::downgrade` from racing to create new - // `Weak` references. `Arc::is_unique` (which backs `Arc::get_mut`) - // needs to observe both the strong and weak counts as indicating - // uniqueness in one logical atomic step; since they live in separate - // atomic words, it locks the weak count while reading the strong - // count to keep the two reads consistent. - weak: Atomic, - - data: T, -} +type ArcHeader = Header>; -/// Calculate layout for `ArcInner` using the inner value's layout -fn arcinner_layout_for_value_layout(layout: Layout) -> Layout { - // Calculate layout using the given value layout. - // Previously, layout was calculated on the expression - // `&*(ptr as *const ArcInner)`, but this created a misaligned - // reference (see #54908). - Layout::new::>() - .extend(layout) - .unwrap_or_else(|_| panic!("capacity overflow")) - .0 - .pad_to_align() -} +// Debug visualizers relies on the these deterministic field offsets. +const _: () = { + assert!(size_of::() - mem::offset_of!(ArcHeader, strong) == size_of::() * 2); + assert!(size_of::() - mem::offset_of!(ArcHeader, weak) == size_of::()); +}; + +#[expect(clippy::declare_interior_mutable_const, reason = "by design")] +impl ArcHeader { + /// Initial header for newly created `Arc` value. The initial weak count of 1 used for tracking + /// all strong counts. + const INIT_ARC: Self = Self { weak: Atomic::::new(1), strong: Atomic::::new(1) }; -unsafe impl Send for ArcInner {} -unsafe impl Sync for ArcInner {} + /// Initial header for newly created `UniqueRc` value. + const INIT_UNIQUE_ARC: Self = + Self { weak: Atomic::::new(1), strong: Atomic::::new(0) }; +} impl Arc { /// Constructs a new `Arc`. @@ -442,15 +413,7 @@ impl Arc { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn new(data: T) -> Arc { - // Start the weak pointer count as 1 which is the weak pointer that's - // held by all the strong pointers (kinda), see std/rc.rs for more info - let x: Box<_> = Box::new(ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data, - }); - // SAFETY: Pointer is valid. - unsafe { Self::from_inner(Box::into_non_null(x)) } + Self::new_in(data, Global) } /// Constructs a new `Arc` while giving you a `Weak` to the allocation, @@ -535,14 +498,7 @@ impl Arc { #[stable(feature = "new_uninit", since = "1.82.0")] #[must_use] pub fn new_uninit() -> Arc> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::from_ptr(Arc::allocate_for_layout( - Layout::new::(), - |layout| Global.allocate(layout), - <*mut u8>::cast, - )) - } + Self::new_uninit_in(Global) } /// Constructs a new `Arc` with uninitialized contents, with the memory @@ -568,14 +524,7 @@ impl Arc { #[stable(feature = "new_zeroed_alloc", since = "1.92.0")] #[must_use] pub fn new_zeroed() -> Arc> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::from_ptr(Arc::allocate_for_layout( - Layout::new::(), - |layout| Global.allocate_zeroed(layout), - <*mut u8>::cast, - )) - } + Self::new_zeroed_in(Global) } /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then @@ -610,15 +559,7 @@ impl Arc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new(data: T) -> Result, AllocError> { - // Start the weak pointer count as 1 which is the weak pointer that's - // held by all the strong pointers (kinda), see std/rc.rs for more info - let x: Box<_> = Box::try_new(ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data, - })?; - // SAFETY: Pointer is valid. - unsafe { Ok(Self::from_inner(Box::into_non_null(x))) } + Self::try_new_in(data, Global) } /// Constructs a new `Arc` with uninitialized contents, returning an error @@ -643,14 +584,7 @@ impl Arc { /// ``` #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_new_uninit() -> Result>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Arc::from_ptr(Arc::try_allocate_for_layout( - Layout::new::(), - |layout| Global.allocate(layout), - <*mut u8>::cast, - )?)) - } + Self::try_new_uninit_in(Global) } /// Constructs a new `Arc` with uninitialized contents, with the memory @@ -676,14 +610,7 @@ impl Arc { /// [zeroed]: mem::MaybeUninit::zeroed #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_new_zeroed() -> Result>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Arc::from_ptr(Arc::try_allocate_for_layout( - Layout::new::(), - |layout| Global.allocate_zeroed(layout), - <*mut u8>::cast, - )?)) - } + Self::try_new_zeroed_in(Global) } } @@ -704,19 +631,15 @@ impl Arc { #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn new_in(data: T, alloc: A) -> Arc { - // Start the weak pointer count as 1 which is the weak pointer that's - // held by all the strong pointers (kinda), see std/rc.rs for more info - let x = Box::new_in( - ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data, - }, - alloc, - ); - let (ptr, alloc) = Box::into_non_null_with_allocator(x); - // SAFETY: Pointer is valid. - unsafe { Self::from_inner_in(ptr, alloc) } + let mut uninit_arc = Self::new_uninit_in(alloc); + + // SAFETY: `uninit_arc` points to a newly allocated reference counted allocation, we have + // exclusive access to the contained value. + unsafe { + Arc::get_mut_unchecked(&mut uninit_arc).write(data); + + uninit_arc.assume_init() + } } /// Constructs a new `Arc` with uninitialized contents in the provided allocator. @@ -745,17 +668,11 @@ impl Arc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_uninit_in(alloc: A) -> Arc, A> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::from_ptr_in( - Arc::allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate(layout), - <*mut u8>::cast, - ), - alloc, - ) - } + let ptr = ArcHeader::allocate_uninit_for_type(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Arc::init_from_parts(ptr, alloc) } } /// Constructs a new `Arc` with uninitialized contents, with the memory @@ -783,17 +700,11 @@ impl Arc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_zeroed_in(alloc: A) -> Arc, A> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::from_ptr_in( - Arc::allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate_zeroed(layout), - <*mut u8>::cast, - ), - alloc, - ) - } + let ptr = ArcHeader::allocate_zeroed_for_type(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Arc::init_from_parts(ptr, alloc) } } /// Constructs a new `Arc` in the given allocator while giving you a `Weak` to the allocation, @@ -834,15 +745,10 @@ impl Arc { { // Construct the inner in the "uninitialized" state with a single // weak reference. - let (uninit_ptr, alloc) = Box::into_non_null_with_allocator(Box::new_in( - ArcInner { - strong: atomic::AtomicUsize::new(0), - weak: atomic::AtomicUsize::new(1), - data: mem::MaybeUninit::::uninit(), - }, - alloc, - )); - let init_ptr: NonNull> = uninit_ptr.cast(); + let init_ptr = ArcHeader::allocate_uninit_for_type(&alloc); + + // SAFETY: We have exclusive access to `init_ptr`. + unsafe { ArcHeader::INIT_UNIQUE_ARC.write_to(init_ptr.erase()) }; let weak = Weak { ptr: init_ptr, alloc }; @@ -854,12 +760,9 @@ impl Arc { // otherwise. let data = data_fn(&weak); - // Now we can properly initialize the inner value and turn our weak - // reference into a strong reference. - let inner = init_ptr.as_ptr(); // ignore-tidy-undocumented-unsafe unsafe { - ptr::write(&raw mut (*inner).data, data); + init_ptr.ptr.write(data); // The above write to the data field must be visible to any threads which // observe a non-zero strong count. Therefore we need at least "Release" ordering @@ -873,7 +776,7 @@ impl Arc { // // These side effects do not impact us in any way, and no other side effects are // possible with safe code alone. - let prev_value = (*inner).strong.fetch_add(1, Release); + let prev_value = ArcHeader::get(init_ptr.erase()).strong.fetch_add(1, Release); debug_assert_eq!(prev_value, 0, "No prior strong references should exist"); // Strong references should collectively own a shared weak reference, @@ -927,19 +830,13 @@ impl Arc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_in(data: T, alloc: A) -> Result, AllocError> { - // Start the weak pointer count as 1 which is the weak pointer that's - // held by all the strong pointers (kinda), see std/rc.rs for more info - let x = Box::try_new_in( - ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data, - }, - alloc, - )?; - let (ptr, alloc) = Box::into_non_null_with_allocator(x); - // SAFETY: Pointer is valid since we created it. - Ok(unsafe { Self::from_inner_in(ptr, alloc) }) + // SAFETY: `uninit_arc` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + Self::try_new_uninit_in(alloc).map(|mut uninit_arc| unsafe { + Arc::get_mut_unchecked(&mut uninit_arc).write(data); + + uninit_arc.assume_init() + }) } /// Constructs a new `Arc` with uninitialized contents, in the provided allocator, returning an @@ -969,17 +866,11 @@ impl Arc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_uninit_in(alloc: A) -> Result, A>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Arc::from_ptr_in( - Arc::try_allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate(layout), - <*mut u8>::cast, - )?, - alloc, - )) - } + ArcHeader::try_allocate_uninit_for_type(&alloc).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Arc::init_from_parts(ptr, alloc) } + }) } /// Constructs a new `Arc` with uninitialized contents, with the memory @@ -1008,17 +899,11 @@ impl Arc { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_zeroed_in(alloc: A) -> Result, A>, AllocError> { - // ignore-tidy-undocumented-unsafe - unsafe { - Ok(Arc::from_ptr_in( - Arc::try_allocate_for_layout( - Layout::new::(), - |layout| alloc.allocate_zeroed(layout), - <*mut u8>::cast, - )?, - alloc, - )) - } + ArcHeader::try_allocate_zeroed_for_type(&alloc).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Arc::init_from_parts(ptr, alloc) } + }) } /// Returns the inner value, if the `Arc` has exactly one strong reference. /// @@ -1065,7 +950,7 @@ impl Arc { let this = ManuallyDrop::new(this); // SAFETY: Pointer is valid for reads, contains initialised memory, // and not dropped multiple times (we return it). - let elem: T = unsafe { ptr::read(&this.ptr.as_ref().data) }; + let elem: T = unsafe { this.ptr.ptr.read() }; // SAFETY: As above, but we explicitly drop the allocator only once // upon creating and dropping a weak pointer. let alloc: A = unsafe { ptr::read(&this.alloc) }; // copy the allocator @@ -1325,8 +1210,7 @@ impl Arc<[T]> { #[stable(feature = "new_uninit", since = "1.82.0")] #[must_use] pub fn new_uninit_slice(len: usize) -> Arc<[mem::MaybeUninit]> { - // ignore-tidy-undocumented-unsafe - unsafe { Arc::from_ptr(Arc::allocate_for_slice(len)) } + Self::new_uninit_slice_in(len, Global) } /// Constructs a new atomically reference-counted slice with uninitialized contents, with the memory being @@ -1352,14 +1236,7 @@ impl Arc<[T]> { #[stable(feature = "new_zeroed_alloc", since = "1.92.0")] #[must_use] pub fn new_zeroed_slice(len: usize) -> Arc<[mem::MaybeUninit]> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::from_ptr(Arc::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| Global.allocate_zeroed(layout), - |mem| mem.cast::().cast_slice(len) as *mut ArcInner<[mem::MaybeUninit]>, - )) - } + Self::new_zeroed_slice_in(len, Global) } } @@ -1393,8 +1270,11 @@ impl Arc<[T], A> { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_uninit_slice_in(len: usize, alloc: A) -> Arc<[mem::MaybeUninit], A> { - // ignore-tidy-undocumented-unsafe - unsafe { Arc::from_ptr_in(Arc::allocate_for_slice_in(len, &alloc), alloc) } + let ptr = ArcHeader::allocate_uninit_slice(&alloc, len); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Arc::init_from_parts(ptr, alloc) } } /// Constructs a new atomically reference-counted slice with uninitialized contents, with the memory being @@ -1422,17 +1302,11 @@ impl Arc<[T], A> { #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Arc<[mem::MaybeUninit], A> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::from_ptr_in( - Arc::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| alloc.allocate_zeroed(layout), - |mem| mem.cast::().cast_slice(len) as *mut ArcInner<[mem::MaybeUninit]>, - ), - alloc, - ) - } + let ptr = ArcHeader::allocate_zeroed_slice(&alloc, len); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Arc::init_from_parts(ptr, alloc) } } /// Converts the reference-counted slice into a reference-counted array. @@ -1558,17 +1432,11 @@ impl Arc { #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn clone_from_ref_in(value: &T, alloc: A) -> Arc { - // `in_progress` drops the allocation if we panic before finishing initializing it. - let mut in_progress: UniqueArcUninit = UniqueArcUninit::new(value, alloc); + let ptr = ArcHeader::allocate_from_cloning(&alloc, value); - // Initialize with clone of value. - // ignore-tidy-undocumented-unsafe - unsafe { - // Clone. If the clone panics, `in_progress` will be dropped and clean up. - value.clone_to_uninit(in_progress.data_ptr().cast()); - // Cast type of pointer, now that it is initialized. - in_progress.into_arc() - } + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, alloc) } } /// Constructs a new `Arc` with a clone of `value` in the provided allocator, returning an error if allocation fails @@ -1587,19 +1455,11 @@ impl Arc { #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn try_clone_from_ref_in(value: &T, alloc: A) -> Result, AllocError> { - // `in_progress` drops the allocation if we panic before finishing initializing it. - let mut in_progress: UniqueArcUninit = UniqueArcUninit::try_new(value, alloc)?; - - // Initialize with clone of value. - // ignore-tidy-undocumented-unsafe - let initialized_clone = unsafe { - // Clone. If the clone panics, `in_progress` will be dropped and clean up. - value.clone_to_uninit(in_progress.data_ptr().cast()); - // Cast type of pointer, now that it is initialized. - in_progress.into_arc() - }; - - Ok(initialized_clone) + ArcHeader::try_allocate_from_cloning(&alloc, value).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have + // exclusive access to it. + unsafe { Self::init_from_parts(ptr, alloc) } + }) } } @@ -1637,9 +1497,10 @@ impl Arc<[mem::MaybeUninit], A> { #[must_use = "`self` will be dropped if the result is not used"] #[inline] pub unsafe fn assume_init(self) -> Arc<[T], A> { - let (ptr, alloc) = Arc::into_inner_with_allocator(self); - // SAFETY: Upheld by caller. - unsafe { Arc::from_ptr_in(ptr.as_ptr() as _, alloc) } + let (ptr, alloc) = Arc::into_raw_with_allocator(self); + + // SAFETY: Caller guarantees the contained value is initialized. + unsafe { Arc::from_raw_in(ptr as _, alloc) } } } @@ -1861,14 +1722,13 @@ impl Arc { #[must_use] #[unstable(feature = "arc_raw_get_strong", issue = "157021")] pub unsafe fn strong_count_from_raw(ptr: *const T) -> usize { - // SAFETY: Upheld by caller. - let offset = unsafe { data_offset(ptr) }; - // Reverse the offset to find the original ArcInner. - // SAFETY: Caller ensures this pointer was to an `Arc` allocation, - // so offsetting must be inbounds. - let arc_ptr = unsafe { ptr.byte_sub(offset) as *mut ArcInner }; - // SAFETY: Per the above, an `ArcInner` is stored here. - unsafe { (*arc_ptr).strong.load(Relaxed) } + // SAFETY: Caller guarantees `ptr` points to a valid reference counted allocation, we are + // safe to get a reference to the header. + let header = unsafe { + ArcHeader::get(ErasedRcValuePointer::new(NonNull::new_unchecked(ptr.cast_mut().cast()))) + }; + + header.strong.load(Relaxed) } } @@ -1932,12 +1792,10 @@ impl Arc { #[stable(feature = "rc_as_ptr", since = "1.45.0")] #[rustc_never_returns_null_ptr] pub fn as_ptr(this: &Self) -> *const T { - let ptr: *mut ArcInner = NonNull::as_ptr(this.ptr); - - // SAFETY: This cannot go through Deref::deref or ArcInnerPtr::inner because + // This cannot go through Deref::deref because // this is required to retain raw/mut provenance such that e.g. `get_mut` can // write through the pointer after the Arc is recovered through `from_raw`. - unsafe { &raw mut (*ptr).data } + this.ptr.as_ptr() } /// Constructs an `Arc` from a raw pointer. @@ -2015,12 +1873,7 @@ impl Arc { pub unsafe fn from_raw_in(ptr: *const T, alloc: A) -> Self { // SAFETY: Upheld by caller. unsafe { - let offset = data_offset(ptr); - - // Reverse the offset to find the original ArcInner. - let arc_ptr = ptr.byte_sub(offset) as *mut ArcInner; - - Self::from_ptr_in(arc_ptr, alloc) + Self::from_inner_in(RcValuePointer::new(NonNull::new_unchecked(ptr.cast_mut())), alloc) } } @@ -2068,7 +1921,7 @@ impl Arc { match this.inner().weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) { Ok(_) => { // Make sure we do not create a dangling Weak - debug_assert!(!is_dangling(this.ptr.as_ptr())); + debug_assert!(!this.ptr.is_dangling()); return Weak { ptr: this.ptr, alloc: this.alloc.clone() }; } Err(old) => cur = old, @@ -2226,13 +2079,10 @@ impl Arc { } #[inline] - fn inner(&self) -> &ArcInner { + fn inner(&self) -> &ArcHeader { // SAFETY: While this arc is alive we're guaranteed - // that the inner pointer is valid. Furthermore, we know that the - // `ArcInner` structure itself is `Sync` if the inner data is - // `Sync` as well, so we're ok loaning out an immutable pointer to these - // contents. - unsafe { self.ptr.as_ref() } + // that the header is valid. + unsafe { ArcHeader::get(self.ptr.erase()) } } // Non-inlined part of `drop`. @@ -2249,7 +2099,7 @@ impl Arc { // allocation itself (there might still be weak pointers lying around). // We cannot use `get_mut_unchecked` here, because `self.alloc` is borrowed. // ignore-tidy-undocumented-unsafe - unsafe { ptr::drop_in_place(&mut (*self.ptr.as_ptr()).data) }; + unsafe { self.ptr.ptr.drop_in_place() }; } /// Returns `true` if the two `Arc`s point to the same allocation in a vein similar to @@ -2277,226 +2127,39 @@ impl Arc { } } -impl Arc { - /// Allocates an `ArcInner` with sufficient space for - /// a possibly-unsized inner value where the value has the layout provided. - /// - /// The function `mem_to_arcinner` is called with the data pointer - /// and must return back a (potentially fat)-pointer for the `ArcInner`. - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_layout( - value_layout: Layout, - allocate: impl FnOnce(Layout) -> Result, AllocError>, - mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner, - ) -> *mut ArcInner { - let layout = arcinner_layout_for_value_layout(value_layout); - - let ptr = allocate(layout).unwrap_or_else(|_| handle_alloc_error(layout)); - - // ignore-tidy-undocumented-unsafe - unsafe { Self::initialize_arcinner(ptr, layout, mem_to_arcinner) } - } - - /// Allocates an `ArcInner` with sufficient space for - /// a possibly-unsized inner value where the value has the layout provided, - /// returning an error if allocation fails. - /// - /// The function `mem_to_arcinner` is called with the data pointer - /// and must return back a (potentially fat)-pointer for the `ArcInner`. - unsafe fn try_allocate_for_layout( - value_layout: Layout, - allocate: impl FnOnce(Layout) -> Result, AllocError>, - mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner, - ) -> Result<*mut ArcInner, AllocError> { - let layout = arcinner_layout_for_value_layout(value_layout); - - let ptr = allocate(layout)?; - - // ignore-tidy-undocumented-unsafe - let inner = unsafe { Self::initialize_arcinner(ptr, layout, mem_to_arcinner) }; - - Ok(inner) - } - - unsafe fn initialize_arcinner( - ptr: NonNull<[u8]>, - layout: Layout, - mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner, - ) -> *mut ArcInner { - let inner = mem_to_arcinner(ptr.as_non_null_ptr().as_ptr()); - // SAFETY: Upheld by caller. - debug_assert_eq!(unsafe { Layout::for_value_raw(inner) }, layout); - - // ignore-tidy-undocumented-unsafe - unsafe { - (&raw mut (*inner).strong).write(atomic::AtomicUsize::new(1)); - (&raw mut (*inner).weak).write(atomic::AtomicUsize::new(1)); - } - - inner - } -} - -impl Arc { - /// Allocates an `ArcInner` with sufficient space for an unsized inner value. - #[inline] - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_ptr_in(ptr: *const T, alloc: &A) -> *mut ArcInner { - // Allocate for the `ArcInner` using the given value. - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::allocate_for_layout( - Layout::for_value_raw(ptr), - |layout| alloc.allocate(layout), - |mem| mem.with_metadata_of(ptr as *const ArcInner), - ) - } - } - - #[cfg(not(no_global_oom_handling))] - fn from_box_in(src: Box) -> Arc { - // ignore-tidy-undocumented-unsafe - unsafe { - let value_size = size_of_val(&*src); - let ptr = Self::allocate_for_ptr_in(&*src, Box::allocator(&src)); - - // Copy value as bytes - ptr::copy_nonoverlapping( - (&raw const *src) as *const u8, - (&raw mut (*ptr).data) as *mut u8, - value_size, - ); - - // Free the allocation without dropping its contents - let (bptr, alloc) = Box::into_raw_with_allocator(src); - let src = Box::from_raw_in(bptr as *mut mem::ManuallyDrop, &alloc); - drop(src); - - Self::from_ptr_in(ptr, alloc) - } - } -} - -impl Arc<[T]> { - /// Allocates an `ArcInner<[T]>` with the given length. - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_slice(len: usize) -> *mut ArcInner<[T]> { - // ignore-tidy-undocumented-unsafe - unsafe { - Self::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| Global.allocate(layout), - |mem| mem.cast::().cast_slice(len) as *mut ArcInner<[T]>, - ) - } - } - - /// Copy elements from slice into newly allocated `Arc<[T]>` - /// - /// Unsafe because the caller must either take ownership, bind `T: Copy` or - /// bind `T: TrivialClone`. - #[cfg(not(no_global_oom_handling))] - unsafe fn copy_from_slice(v: &[T]) -> Arc<[T]> { - // ignore-tidy-undocumented-unsafe - unsafe { - let ptr = Self::allocate_for_slice(v.len()); - - ptr::copy_nonoverlapping(v.as_ptr(), (&raw mut (*ptr).data) as *mut T, v.len()); - - Self::from_ptr(ptr) - } - } - - /// Constructs an `Arc<[T]>` from an iterator known to be of a certain size. - /// - /// Behavior is undefined should the size be wrong. - #[cfg(not(no_global_oom_handling))] - unsafe fn from_iter_exact(iter: impl Iterator, len: usize) -> Arc<[T]> { - // Panic guard while cloning T elements. - // In the event of a panic, elements that have been written - // into the new ArcInner will be dropped, then the memory freed. - struct Guard { - mem: NonNull, - elems: *mut T, - layout: Layout, - n_elems: usize, - } - - impl Drop for Guard { - fn drop(&mut self) { - // ignore-tidy-undocumented-unsafe - unsafe { - let slice = from_raw_parts_mut(self.elems, self.n_elems); - ptr::drop_in_place(slice); - - Global.deallocate(self.mem, self.layout); - } - } - } - - // ignore-tidy-undocumented-unsafe - unsafe { - let ptr = Self::allocate_for_slice(len); - - let mem = ptr as *mut _ as *mut u8; - let layout = Layout::for_value_raw(ptr); - - // Pointer to first element - let elems = (&raw mut (*ptr).data) as *mut T; - - let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 }; - - for (i, item) in iter.enumerate() { - ptr::write(elems.add(i), item); - guard.n_elems += 1; - } - - // All clear. Forget the guard so it doesn't free the new ArcInner. - mem::forget(guard); - - Self::from_ptr(ptr) - } - } -} - -impl Arc<[T], A> { - /// Allocates an `ArcInner<[T]>` with the given length. - #[inline] - #[cfg(not(no_global_oom_handling))] - unsafe fn allocate_for_slice_in(len: usize, alloc: &A) -> *mut ArcInner<[T]> { - // ignore-tidy-undocumented-unsafe - unsafe { - Arc::allocate_for_layout( - Layout::array::(len).unwrap(), - |layout| alloc.allocate(layout), - |mem| mem.cast::().cast_slice(len) as *mut ArcInner<[T]>, - ) - } - } -} - -/// Specialization trait used for `From<&[T]>`. -#[cfg(not(no_global_oom_handling))] -trait ArcFromSlice { - fn from_slice(slice: &[T]) -> Self; -} - -#[cfg(not(no_global_oom_handling))] -impl ArcFromSlice for Arc<[T]> { - #[inline] - default fn from_slice(v: &[T]) -> Self { - // ignore-tidy-undocumented-unsafe - unsafe { Self::from_iter_exact(v.iter().cloned(), v.len()) } - } -} - -#[cfg(not(no_global_oom_handling))] -impl ArcFromSlice for Arc<[T]> { - #[inline] - fn from_slice(v: &[T]) -> Self { - // SAFETY: `T` implements `TrivialClone`, so this is sound and equivalent - // to the above. - unsafe { Arc::copy_from_slice(v) } +unsafe fn inc_strong(ptr: ErasedRcValuePointer) { + // Using a relaxed ordering is alright here, as knowledge of the + // original reference prevents other threads from erroneously deleting + // the object. + // + // As explained in the [Boost documentation][1], Increasing the + // reference counter can always be done with memory_order_relaxed: New + // references to an object can only be formed from an existing + // reference, and passing an existing reference from one thread to + // another must already provide any required synchronization. + // + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) + + // SAFETY: Caller guarantees the validity of `ptr`. + let old_size = unsafe { ArcHeader::get(ptr) }.strong.fetch_add(1, Relaxed); + + // However we need to guard against massive refcounts in case someone is `mem::forget`ing + // Arcs. If we don't do this the count can overflow and users will use-after free. This + // branch will never be taken in any realistic program. We abort because such a program is + // incredibly degenerate, and we don't care to support it. + // + // This check is not 100% water-proof: we error when the refcount grows beyond `isize::MAX`. + // But we do that check *after* having done the increment, so there is a chance here that + // the worst already happened and we actually do overflow the `usize` counter. However, that + // requires the counter to grow from `isize::MAX` to `usize::MAX` between the increment + // above and the `abort` below, which seems exceedingly unlikely. + // + // This is a global invariant, and also applies when using a compare-exchange loop to increment + // counters in other methods. + // Otherwise, the counter could be brought to an almost-overflow using a compare-exchange loop, + // and then overflow using a few `fetch_add`s. + if old_size > MAX_REFCOUNT { + intrinsics::abort_immediate(); } } @@ -2518,40 +2181,12 @@ impl Clone for Arc { /// ``` #[inline] fn clone(&self) -> Arc { - // Using a relaxed ordering is alright here, as knowledge of the - // original reference prevents other threads from erroneously deleting - // the object. - // - // As explained in the [Boost documentation][1], Increasing the - // reference counter can always be done with memory_order_relaxed: New - // references to an object can only be formed from an existing - // reference, and passing an existing reference from one thread to - // another must already provide any required synchronization. - // - // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) - let old_size = self.inner().strong.fetch_add(1, Relaxed); + // SAFETY: Pointer is valid & allocator corresponds to the one used to allocate it. + unsafe { + inc_strong(self.ptr.erase()); - // However we need to guard against massive refcounts in case someone is `mem::forget`ing - // Arcs. If we don't do this the count can overflow and users will use-after free. This - // branch will never be taken in any realistic program. We abort because such a program is - // incredibly degenerate, and we don't care to support it. - // - // This check is not 100% water-proof: we error when the refcount grows beyond `isize::MAX`. - // But we do that check *after* having done the increment, so there is a chance here that - // the worst already happened and we actually do overflow the `usize` counter. However, that - // requires the counter to grow from `isize::MAX` to `usize::MAX` between the increment - // above and the `abort` below, which seems exceedingly unlikely. - // - // This is a global invariant, and also applies when using a compare-exchange loop to increment - // counters in other methods. - // Otherwise, the counter could be brought to an almost-overflow using a compare-exchange loop, - // and then overflow using a few `fetch_add`s. - if old_size > MAX_REFCOUNT { - intrinsics::abort_immediate(); + Self::from_inner_in(self.ptr, self.alloc.clone()) } - - // SAFETY: Pointer is valid & allocator corresponds to the one used to allocate it. - unsafe { Self::from_inner_in(self.ptr, self.alloc.clone()) } } } @@ -2567,7 +2202,8 @@ impl Deref for Arc { #[inline] fn deref(&self) -> &T { - &self.inner().data + // SAFETY: `Arc` always contains a valid value. + unsafe { self.ptr.ptr.as_ref() } } } @@ -2651,7 +2287,7 @@ impl Arc { // // Use Acquire to ensure that we see any writes to `weak` that happen // before release writes (i.e., decrements) to `strong`. Since we hold a - // weak count, there's no chance the ArcInner itself could be + // weak count, there's no chance the allocation itself could be // deallocated. if this.inner().strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() { // Another strong pointer exists, so we must clone. @@ -2673,10 +2309,10 @@ impl Arc { // If we unwind before the Arc is overwritten, we expose a strong // count of 0, resulting in a UAF (#155746, #157203). // Until the new Arc is written, the old Arc must remain valid - struct Guard<'a, T: ?Sized> { - inner: &'a ArcInner, + struct Guard<'a> { + inner: &'a ArcHeader, } - impl<'a, T: ?Sized> Drop for Guard<'a, T> { + impl<'a> Drop for Guard<'a> { fn drop(&mut self) { self.inner.strong.store(1, Release); } @@ -2688,7 +2324,7 @@ impl Arc { // - The allocation can fail // - The allocator clone can fail let mut in_progress: UniqueArcUninit = - UniqueArcUninit::new(&**this, this.alloc.clone()); + UniqueArcUninit::new(this, this.alloc.clone()); // ignore-tidy-undocumented-unsafe unsafe { @@ -2705,7 +2341,7 @@ impl Arc { mem::forget(guard); // Materialize our own implicit weak pointer, so that it can clean - // up the ArcInner as needed. + // up the allocation as needed. // Make sure the allocator is not leaked when the Arc is overwritten. // Only drop at the end of the scope to avoid panics. let _weak = Weak { ptr: this.ptr, alloc: ptr::read(&this.alloc) }; @@ -2866,7 +2502,7 @@ impl Arc { // We are careful to *not* create a reference covering the "count" fields, as // this would alias with concurrent access to the reference counts (e.g. by `Weak`). // ignore-tidy-undocumented-unsafe - unsafe { &mut (*this.ptr.as_ptr()).data } + unsafe { this.ptr.ptr.as_mut() } } /// Determine whether this is the unique reference to the underlying data. @@ -3020,7 +2656,7 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Arc { // Make sure we aren't trying to "drop" the shared static for empty slices // used by Default::default. debug_assert!( - !ptr::addr_eq(self.ptr.as_ptr(), &STATIC_INNER_SLICE.inner), + !ptr::addr_eq(self.ptr.as_ptr(), STATIC_INNER_SLICE_VALUE_PTR.ptr.as_ptr()), "Arcs backed by a static should never reach a strong count of 0. \ Likely decrement_strong_count or from_raw were called too many times.", ); @@ -3127,7 +2763,7 @@ impl Weak { #[rustc_const_stable(feature = "const_weak_new", since = "1.73.0")] #[must_use] pub const fn new() -> Weak { - Weak { ptr: NonNull::without_provenance(NonZeroUsize::MAX), alloc: Global } + Weak { ptr: RcValuePointer::DANGLING, alloc: Global } } } @@ -3152,17 +2788,10 @@ impl Weak { #[inline] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn new_in(alloc: A) -> Weak { - Weak { ptr: NonNull::without_provenance(NonZeroUsize::MAX), alloc } + Weak { ptr: RcValuePointer::DANGLING, alloc } } } -/// Helper type to allow accessing the reference counts without -/// making any assertions about the data field. -struct WeakInner<'a> { - weak: &'a Atomic, - strong: &'a Atomic, -} - impl Weak { /// Converts a raw pointer previously created by [`into_raw`] back into `Weak`. /// @@ -3282,18 +2911,7 @@ impl Weak { #[must_use] #[stable(feature = "weak_into_raw", since = "1.45.0")] pub fn as_ptr(&self) -> *const T { - let ptr: *mut ArcInner = NonNull::as_ptr(self.ptr); - - if is_dangling(ptr) { - // If the pointer is dangling, we return the sentinel directly. This cannot be - // a valid payload address, as the payload is at least as aligned as ArcInner (usize). - ptr as *const T - } else { - // SAFETY: if is_dangling returns false, then the pointer is dereferenceable. - // The payload may be dropped at this point, and we have to maintain provenance, - // so use raw pointer manipulation. - unsafe { &raw mut (*ptr).data } - } + self.ptr.as_ptr() } /// Consumes the `Weak`, returning the wrapped pointer and allocator. @@ -3380,22 +2998,8 @@ impl Weak { #[inline] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub unsafe fn from_raw_in(ptr: *const T, alloc: A) -> Self { - // See Weak::as_ptr for context on how the input pointer is derived. - - let ptr = if is_dangling(ptr) { - // This is a dangling Weak. - ptr as *mut ArcInner - } else { - // Otherwise, we're guaranteed the pointer came from a nondangling Weak. - // SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T. - let offset = unsafe { data_offset(ptr) }; - // Thus, we reverse the offset to get the whole ArcInner. - // SAFETY: the pointer originated from a Weak, so this offset is safe. - unsafe { ptr.byte_sub(offset) as *mut ArcInner } - }; - - // SAFETY: we now have recovered the original Weak pointer, so can create the Weak. - Weak { ptr: unsafe { NonNull::new_unchecked(ptr) }, alloc } + // SAFETY: Caller guarantees the validity of `ptr`. + Weak { ptr: RcValuePointer::new(unsafe { NonNull::new_unchecked(ptr.cast_mut()) }), alloc } } } @@ -3506,19 +3110,20 @@ impl Weak { } } - /// Returns `None` when the pointer is dangling and there is no allocated `ArcInner`, + /// Returns `None` when the pointer is dangling and there is no allocated value, /// (i.e., when this `Weak` was created by `Weak::new`). #[inline] - fn inner(&self) -> Option> { - let ptr = self.ptr.as_ptr(); - if is_dangling(ptr) { + fn inner(&self) -> Option<&ArcHeader> { + let ptr = self.ptr; + + if ptr.is_dangling() { None } else { // We are careful to *not* create a reference covering the "data" field, as // the field may be mutated concurrently (for example, if the last `Arc` // is dropped, the data field will be dropped in-place). // ignore-tidy-undocumented-unsafe - Some(unsafe { WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak } }) + Some(unsafe { ArcHeader::get(ptr.erase()) }) } } @@ -3670,15 +3275,13 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Weak { // Make sure we aren't trying to "deallocate" the shared static for empty slices // used by Default::default. debug_assert!( - !ptr::addr_eq(self.ptr.as_ptr(), &STATIC_INNER_SLICE.inner), + !ptr::addr_eq(self.ptr.as_ptr(), STATIC_INNER_SLICE_VALUE_PTR.ptr.as_ptr()), "Arc/Weaks backed by a static should never be deallocated. \ Likely decrement_strong_count or from_raw were called too many times.", ); // ignore-tidy-undocumented-unsafe - unsafe { - self.alloc.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr())) - } + unsafe { ArcHeader::deallocate(&self.alloc, self.ptr) }; } } } @@ -3913,21 +3516,16 @@ impl Default for Arc { /// assert_eq!(*x, 0); /// ``` fn default() -> Arc { - // ignore-tidy-undocumented-unsafe - unsafe { - Self::from_inner(Box::into_non_null(Box::write( - Box::new_uninit(), - ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data: T::default(), - }, - ))) - } + let alloc = Global; + let ptr = ArcHeader::allocate_from_default(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation with value + // initialized. + unsafe { Self::init_from_parts(ptr, alloc) } } } -/// Struct to hold the static `ArcInner` used for empty `Arc` as +/// Struct to hold the static reference counted allocation used for empty `Arc` as /// returned by `Default::default`. /// /// Layout notes: @@ -3936,19 +3534,53 @@ impl Default for Arc { /// * `[u8; 1]` (to be initialized with 0) so it can be used for `Arc`. #[repr(C, align(16))] struct SliceArcInnerForStatic { - inner: ArcInner<[u8; 1]>, + _padding: MaybeUninit< + [u8; size_of::().next_multiple_of(MAX_STATIC_INNER_SLICE_ALIGNMENT) + - size_of::()], + >, + header: ArcHeader, + value: [u8; 1], } -#[cfg(not(no_global_oom_handling))] + const MAX_STATIC_INNER_SLICE_ALIGNMENT: usize = 16; static STATIC_INNER_SLICE: SliceArcInnerForStatic = SliceArcInnerForStatic { - inner: ArcInner { - strong: atomic::AtomicUsize::new(1), - weak: atomic::AtomicUsize::new(1), - data: [0], - }, + _padding: MaybeUninit::uninit(), + header: ArcHeader::INIT_ARC, + value: [0], }; +// SAFETY: `offset_of!` gives us the correct offset to the `value` field. We use pointer arithmetic +// to preserve the provenance for header accesses. +const STATIC_INNER_SLICE_VALUE_PTR: ErasedRcValuePointer = ErasedRcValuePointer::new(unsafe { + NonNull::from_ref(&STATIC_INNER_SLICE) + .cast() + .byte_add(mem::offset_of!(SliceArcInnerForStatic, value)) +}); + +/// # Safety +/// +/// The resulting value must satisfy the following conditions: +/// +/// - Its size must not exceed 1 byte. +/// - Its alignment must not exceed `MAX_STATIC_INNER_SLICE_ALIGNMENT`. +/// - It must not be interior mutable. +/// - It is valid to initialize the value with all zero bytes. +#[cfg(not(no_global_oom_handling))] +unsafe fn clone_static_arc(metadata: ::Metadata, alloc: A) -> Arc +where + T: ?Sized, + A: Allocator, +{ + // SAFETY: `SliceArcInnerForStatic` is designed to hold a valid reference counted + // allocation, and caller guarantees that the value can be shared as `T`. + unsafe { + inc_strong(STATIC_INNER_SLICE_VALUE_PTR); + + Arc::from_inner_in(STATIC_INNER_SLICE_VALUE_PTR.unerase_with(metadata), alloc) + } +} + #[cfg(not(no_global_oom_handling))] #[stable(feature = "more_rc_default_impls", since = "1.80.0")] impl Default for Arc { @@ -3961,7 +3593,7 @@ impl Default for Arc { debug_assert!(core::str::from_utf8(&arc).is_ok()); let (ptr, alloc) = Arc::into_inner_with_allocator(arc); // ignore-tidy-undocumented-unsafe - unsafe { Arc::from_ptr_in(ptr.as_ptr() as *mut ArcInner, alloc) } + unsafe { Arc::from_raw_in(ptr.as_ptr() as *const str, alloc) } } } @@ -3973,15 +3605,8 @@ impl Default for Arc { /// This may or may not share an allocation with other Arcs. #[inline] fn default() -> Self { - use core::ffi::CStr; - let inner: NonNull> = NonNull::from(&STATIC_INNER_SLICE.inner); - let inner: NonNull> = - NonNull::new(inner.as_ptr() as *mut ArcInner).unwrap(); - // `this` semantically is the Arc "owned" by the static, so make sure not to drop it. - let this: mem::ManuallyDrop> = - // ignore-tidy-undocumented-unsafe - unsafe { mem::ManuallyDrop::new(Arc::from_inner(inner)) }; - (*this).clone() + // SAFETY: `CStr` requires a zero byte, which can be provided by the static `Arc`. + unsafe { clone_static_arc(1, Global) } } } @@ -3994,17 +3619,9 @@ impl Default for Arc<[T]> { #[inline] fn default() -> Self { if align_of::() <= MAX_STATIC_INNER_SLICE_ALIGNMENT { - // We take a reference to the whole struct instead of the ArcInner<[u8; 1]> inside it so - // we don't shrink the range of bytes the ptr is allowed to access under Stacked Borrows. - // (Miri complains on 32-bit targets with Arc<[Align16]> otherwise.) - // (Note that NonNull::from(&STATIC_INNER_SLICE.inner) is fine under Tree Borrows.) - let inner: NonNull = NonNull::from(&STATIC_INNER_SLICE); - let inner: NonNull> = inner.cast(); - // `this` semantically is the Arc "owned" by the static, so make sure not to drop it. - let this: mem::ManuallyDrop> = - // ignore-tidy-undocumented-unsafe - unsafe { mem::ManuallyDrop::new(Arc::from_inner(inner)) }; - return (*this).clone(); + // SAFETY: We are creating an empty slice, which have size 0, and we have checked the + // alignment of `T` does not exceed what `clone_static_arc` requires. + return unsafe { clone_static_arc(0, Global) }; } // If T's alignment is too large for the static, make a new unique allocation. @@ -4092,7 +3709,7 @@ impl From<&[T]> for Arc<[T]> { /// ``` #[inline] fn from(v: &[T]) -> Arc<[T]> { - >::from_slice(v) + Arc::clone_from_ref_in(v, Global) } } @@ -4190,7 +3807,11 @@ impl From> for Arc { /// ``` #[inline] fn from(v: Box) -> Arc { - Arc::from_box_in(v) + let (ptr, alloc) = ArcHeader::allocate_from_box(v); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, alloc) } } } @@ -4209,19 +3830,11 @@ impl From> for Arc<[T], A> { /// ``` #[inline] fn from(v: Vec) -> Arc<[T], A> { - // ignore-tidy-undocumented-unsafe - unsafe { - let (vec_ptr, len, cap, alloc) = v.into_raw_parts_with_allocator(); - - let rc_ptr = Self::allocate_for_slice_in(len, &alloc); - ptr::copy_nonoverlapping(vec_ptr, (&raw mut (*rc_ptr).data) as *mut T, len); - - // Create a `Vec` with length 0, to deallocate the buffer - // without dropping its contents or the allocator - let _ = Vec::from_raw_parts_in(vec_ptr, 0, cap, &alloc); + let (ptr, alloc) = ArcHeader::allocate_from_vec(v); - Self::from_ptr_in(rc_ptr, alloc) - } + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, alloc) } } } @@ -4328,45 +3941,11 @@ impl FromIterator for Arc<[T]> { /// # assert_eq!(&*evens, &*(0..10).collect::>()); /// ``` fn from_iter>(iter: I) -> Self { - ToArcSlice::to_arc_slice(iter.into_iter()) - } -} - -#[cfg(not(no_global_oom_handling))] -/// Specialization trait used for collecting into `Arc<[T]>`. -trait ToArcSlice: Iterator + Sized { - fn to_arc_slice(self) -> Arc<[T]>; -} - -#[cfg(not(no_global_oom_handling))] -impl> ToArcSlice for I { - default fn to_arc_slice(self) -> Arc<[T]> { - self.collect::>().into() - } -} + let ptr = ArcHeader::allocate_from_iter(iter.into_iter()); -#[cfg(not(no_global_oom_handling))] -impl> ToArcSlice for I { - fn to_arc_slice(self) -> Arc<[T]> { - // This is the case for a `TrustedLen` iterator. - let (low, high) = self.size_hint(); - if let Some(high) = high { - debug_assert_eq!( - low, - high, - "TrustedLen iterator's size hint is not exact: {:?}", - (low, high) - ); - - // SAFETY: We need to ensure that the iterator has an exact length and we have. - unsafe { Arc::from_iter_exact(self, low) } - } else { - // TrustedLen contract guarantees that `upper_bound == None` implies an iterator - // length exceeding `usize::MAX`. - // The default implementation would collect into a vec which would panic. - // Thus we panic here immediately without invoking `Vec` code. - panic!("capacity overflow"); - } + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { Self::init_from_parts(ptr, Global) } } } @@ -4387,74 +3966,36 @@ impl AsRef for Arc { #[stable(feature = "pin", since = "1.33.0")] impl Unpin for Arc {} -/// Gets the offset within an `ArcInner` for the payload behind a pointer. -/// -/// # Safety -/// -/// The pointer must point to (and have valid metadata for) a previously -/// valid instance of T, but the T is allowed to be dropped. -unsafe fn data_offset(ptr: *const T) -> usize { - // Align the unsized value to the end of the ArcInner. - // Because ArcInner is repr(C), it will always be the last field in memory. - // SAFETY: since the only unsized types possible are slices, trait objects, - // and extern types, the input safety requirement is currently enough to - // satisfy the requirements of Alignment::of_val_raw; this is an implementation - // detail of the language that must not be relied upon outside of std. - unsafe { data_offset_alignment(Alignment::of_val_raw(ptr)) } -} - -#[inline] -fn data_offset_alignment(alignment: Alignment) -> usize { - let layout = Layout::new::>(); - layout.size() + layout.padding_needed_for(alignment) -} - -/// A unique owning pointer to an [`ArcInner`] **that does not imply the contents are initialized,** -/// but will deallocate it (without dropping the value) when dropped. +/// A unique owning pointer to a reference counted allocation **that does not imply the contents are +/// initialized,** but will deallocate it (without dropping the value) when dropped. /// /// This is a helper for [`Arc::make_mut()`] to ensure correct cleanup on panic. +#[cfg(not(no_global_oom_handling))] struct UniqueArcUninit { - ptr: NonNull>, - layout_for_value: Layout, - alloc: Option, + ptr: RcValuePointer, + alloc: A, } +#[cfg(not(no_global_oom_handling))] impl UniqueArcUninit { - /// Allocates an ArcInner with layout suitable to contain `for_value` or a clone of it. - #[cfg(not(no_global_oom_handling))] - fn new(for_value: &T, alloc: A) -> UniqueArcUninit { - let layout = Layout::for_value(for_value); - // ignore-tidy-undocumented-unsafe - let ptr = unsafe { - Arc::allocate_for_layout( - layout, - |layout_for_arcinner| alloc.allocate(layout_for_arcinner), - |mem| mem.with_metadata_of(ptr::from_ref(for_value) as *const ArcInner), - ) - }; - Self { ptr: NonNull::new(ptr).unwrap(), layout_for_value: layout, alloc: Some(alloc) } - } + /// Allocates a reference counted allocation with layout suitable to contain `for_value` or a + /// clone of it. + fn new(arc: &Arc, alloc: A) -> UniqueArcUninit { + let ptr = + // SAFETY: `Arc` proves that `T` is small enough to store inside a reference + // counted allocation. + unsafe { ArcHeader::allocate_uninit_for_value_unchecked(&alloc, Arc::as_ref(arc)) }; - /// Allocates an ArcInner with layout suitable to contain `for_value` or a clone of it, - /// returning an error if allocation fails. - fn try_new(for_value: &T, alloc: A) -> Result, AllocError> { - let layout = Layout::for_value(for_value); - // ignore-tidy-undocumented-unsafe - let ptr = unsafe { - Arc::try_allocate_for_layout( - layout, - |layout_for_arcinner| alloc.allocate(layout_for_arcinner), - |mem| mem.with_metadata_of(ptr::from_ref(for_value) as *const ArcInner), - )? - }; - Ok(Self { ptr: NonNull::new(ptr).unwrap(), layout_for_value: layout, alloc: Some(alloc) }) + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { ArcHeader::INIT_ARC.write_to(ptr.erase()) }; + + Self { ptr, alloc } } /// Returns the pointer to be written into to initialize the [`Arc`]. fn data_ptr(&mut self) -> *mut T { - let offset = data_offset_alignment(self.layout_for_value.alignment()); - // ignore-tidy-undocumented-unsafe - unsafe { self.ptr.as_ptr().byte_add(offset) as *mut T } + self.ptr.as_ptr() } /// Upgrade this into a normal [`Arc`]. @@ -4463,27 +4004,25 @@ impl UniqueArcUninit { /// /// The data must have been initialized (by writing to [`Self::data_ptr()`]). unsafe fn into_arc(self) -> Arc { - let mut this = ManuallyDrop::new(self); - let ptr = this.ptr.as_ptr(); - let alloc = this.alloc.take().unwrap(); + let this = ManuallyDrop::new(self); + let ptr = this.ptr; + + // SAFETY: We are safe to take ownership of the allocator. + let alloc = unsafe { ptr::read(&this.alloc) }; // SAFETY: The pointer is valid as per `UniqueArcUninit::new`, and the caller is responsible // for having initialized the data. - unsafe { Arc::from_ptr_in(ptr, alloc) } + unsafe { Arc::from_inner_in(ptr, alloc) } } } +#[cfg(not(no_global_oom_handling))] impl Drop for UniqueArcUninit { fn drop(&mut self) { // SAFETY: // * new() produced a pointer safe to deallocate. // * We own the pointer unless into_arc() was called, which forgets us. - unsafe { - self.alloc.take().unwrap().deallocate( - self.ptr.cast(), - arcinner_layout_for_value_layout(self.layout_for_value), - ); - } + unsafe { ArcHeader::deallocate(&self.alloc, self.ptr) }; } } @@ -4543,9 +4082,9 @@ pub struct UniqueArc< T: ?Sized, #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] A: Allocator = Global, > { - ptr: NonNull>, - // Define the ownership of `ArcInner` for drop-check - _marker: PhantomData>, + ptr: RcValuePointer, + // Define the ownership of `T` for drop-check + _marker: PhantomData, // Invariance is necessary for soundness: once other `Weak` // references exist, we already have a form of shared mutability! _marker2: PhantomData<*mut T>, @@ -4799,7 +4338,7 @@ impl UniqueArc { } /// Like [`new`](Self::new), but returns an error if the allocation - /// fails, instead of calling [`handle_alloc_error`]. + /// fails, instead of calling [`handle_alloc_error`](crate::alloc::handle_alloc_error). #[unstable(feature = "unique_rc_arc", issue = "112566")] pub fn try_new(value: T) -> Result { Self::try_new_in(value, Global) @@ -4818,35 +4357,33 @@ impl UniqueArc { // #[unstable(feature = "allocator_api", issue = "163177")] #[must_use] pub fn new_in(data: T, alloc: A) -> Self { - let (ptr, alloc) = Box::into_non_null_with_allocator(Box::new_in( - ArcInner { - strong: atomic::AtomicUsize::new(0), - // keep one weak reference so if all the weak pointers that are created are dropped - // the UniqueArc still stays valid. - weak: atomic::AtomicUsize::new(1), - data, - }, - alloc, - )); + let ptr = ArcHeader::allocate_uninit_for_type(&alloc); + + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { + ArcHeader::INIT_UNIQUE_ARC.write_to(ptr.erase()); + ptr.ptr.write(data); + } + Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc } } /// Like [`new_in`](Self::new_in), but returns an error if the allocation - /// fails, instead of calling [`handle_alloc_error`]. + /// fails, instead of calling [`handle_alloc_error`](crate::alloc::handle_alloc_error). #[unstable(feature = "unique_rc_arc", issue = "112566")] // #[unstable(feature = "allocator_api", issue = "163177")] pub fn try_new_in(data: T, alloc: A) -> Result { - let (ptr, alloc) = Box::into_non_null_with_allocator(Box::try_new_in( - ArcInner { - strong: atomic::AtomicUsize::new(0), - // keep one weak reference so if all the weak pointers that are created are dropped - // the UniqueArc still stays valid. - weak: atomic::AtomicUsize::new(1), - data, - }, - alloc, - )?); - Ok(Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc }) + ArcHeader::try_allocate_uninit_for_type(&alloc).map(|ptr| { + // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive + // access to it. + unsafe { + ArcHeader::INIT_UNIQUE_ARC.write_to(ptr.erase()); + ptr.ptr.write(data); + } + + Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc } + }) } /// Consumes the `UniqueArc`, returning its wrapped value and allocator. @@ -4972,16 +4509,9 @@ impl UniqueArc { impl UniqueArc { #[cfg(not(no_global_oom_handling))] unsafe fn from_raw_with_allocator(ptr: *const T, alloc: A) -> Self { - // SAFETY: Upheld by caller. - let offset = unsafe { data_offset(ptr) }; - - // Reverse the offset to find the original ArcInner. - // SAFETY: Upheld by caller. - let rc_ptr = unsafe { ptr.byte_sub(offset) as *mut ArcInner }; - Self { // SAFETY: Upheld by caller. - ptr: unsafe { NonNull::new_unchecked(rc_ptr) }, + ptr: RcValuePointer::new(unsafe { NonNull::new_unchecked(ptr.cast_mut()) }), _marker: PhantomData, _marker2: PhantomData, alloc, @@ -5014,7 +4544,7 @@ impl UniqueArc { // SAFETY: This pointer was allocated at creation time so we know it is valid. unsafe { // Convert our weak reference into a strong reference - (*this.ptr.as_ptr()).strong.store(1, Release); + ArcHeader::get(this.ptr.erase()).strong.store(1, Release); Arc::from_inner_in(this.ptr, alloc) } } @@ -5025,29 +4555,24 @@ impl UniqueArc { } #[cfg(not(no_global_oom_handling))] - fn inner(&self) -> &ArcInner { + fn inner(&self) -> &ArcHeader { // SAFETY: while this UniqueArc is alive we're guaranteed that the inner pointer is valid. - unsafe { self.ptr.as_ref() } + unsafe { ArcHeader::get(self.ptr.erase()) } } fn as_ptr(this: &Self) -> *const T { - let ptr: *mut ArcInner = NonNull::as_ptr(this.ptr); - - // SAFETY: This cannot go through Deref::deref or UniqueArc::inner because - // this is required to retain raw/mut provenance such that e.g. `get_mut` can - // write through the pointer after the Rc is recovered through `from_raw`. - unsafe { &raw mut (*ptr).data } + this.ptr.as_ptr() } #[inline] - fn into_inner_with_allocator(this: Self) -> (NonNull>, A) { + fn into_inner_with_allocator(this: Self) -> (RcValuePointer, A) { let this = mem::ManuallyDrop::new(this); // SAFETY: Pointer is valid for reads and only read once. (this.ptr, unsafe { ptr::read(&this.alloc) }) } #[inline] - unsafe fn from_inner_in(ptr: NonNull>, alloc: A) -> Self { + unsafe fn from_inner_in(ptr: RcValuePointer, alloc: A) -> Self { Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc } } } @@ -5069,7 +4594,7 @@ impl UniqueArc { // the weak counter. // // SAFETY: This pointer was allocated at creation time so we know it is valid. - let old_size = unsafe { (*this.ptr.as_ptr()).weak.fetch_add(1, Relaxed) }; + let old_size = unsafe { ArcHeader::get(this.ptr.erase()).weak.fetch_add(1, Relaxed) }; // See comments in Arc::clone() for why we do this (for mem::forget). if old_size > MAX_REFCOUNT { @@ -5121,7 +4646,7 @@ impl Deref for UniqueArc { fn deref(&self) -> &T { // SAFETY: This pointer was allocated at creation time so we know it is valid. - unsafe { &self.ptr.as_ref().data } + unsafe { self.ptr.ptr.as_ref() } } } @@ -5136,9 +4661,9 @@ impl DerefMut for UniqueArc { // have unique ownership and therefore it's safe to make a mutable reference because // `UniqueArc` owns the only strong reference to itself. // We also need to be careful to only create a mutable reference to the `data` field, - // as a mutable reference to the entire `ArcInner` would assert uniqueness over the + // as a mutable reference to the entire allocation would assert uniqueness over the // ref count fields too, invalidating any attempt by `Weak`s to access the ref count. - unsafe { &mut (*self.ptr.as_ptr()).data } + unsafe { self.ptr.ptr.as_mut() } } } @@ -5154,7 +4679,7 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for UniqueArc { let _weak = Weak { ptr: self.ptr, alloc: &self.alloc }; // ignore-tidy-undocumented-unsafe - unsafe { ptr::drop_in_place(&mut (*self.ptr.as_ptr()).data) }; + unsafe { self.ptr.ptr.drop_in_place() }; } } diff --git a/src/etc/gdb_providers.py b/src/etc/gdb_providers.py index 0b6458bf20e81..1dad0e5b2eae6 100644 --- a/src/etc/gdb_providers.py +++ b/src/etc/gdb_providers.py @@ -237,33 +237,21 @@ def __init__(self, valobj, is_atomic=False): self._valobj = valobj self._is_atomic = is_atomic - if is_atomic: - self._ptr = unwrap_unique_or_non_null(valobj["ptr"]) - self._value = self._ptr["data"] - # FIXME(shua): the debuginfo template type should be 'str' not 'u8' - if self._ptr.type.target().name == "alloc::rc::RcInner": - length = self._valobj["ptr"]["pointer"]["length"] - u8_ptr_ty = gdb.Type.pointer(gdb.lookup_type("u8")) - ptr = self._value.address.reinterpret_cast(u8_ptr_ty) - self._value = ptr.lazy_string(encoding="utf-8", length=length) - self._strong = unwrap_scalar_wrappers(self._ptr["strong"]) - self._weak = unwrap_scalar_wrappers(self._ptr["weak"]) - 1 - else: - self._ptr = unwrap_unique_or_non_null(valobj["ptr"]["ptr"]) + self._ptr = unwrap_unique_or_non_null(valobj["ptr"]["ptr"]) - if self._ptr.type.name == "*const str": - self._value = self._ptr["data_ptr"].lazy_string( - encoding="utf-8", length=self._ptr["length"] - ) - else: - self._value = self._ptr.dereference() + if self._ptr.type.name == "*const str": + self._value = self._ptr["data_ptr"].lazy_string( + encoding="utf-8", length=self._ptr["length"] + ) + else: + self._value = self._ptr.dereference() - usize_ptr_type = _get_usize_ptr_type() - weak_ptr = self._ptr.reinterpret_cast(usize_ptr_type) - 1 - strong_ptr = weak_ptr - 1 + usize_ptr_type = _get_usize_ptr_type() + weak_ptr = self._ptr.reinterpret_cast(usize_ptr_type) - 1 + strong_ptr = weak_ptr - 1 - self._strong = strong_ptr.dereference() - self._weak = weak_ptr.dereference() - 1 + self._strong = strong_ptr.dereference() + self._weak = weak_ptr.dereference() - 1 def to_string(self): if self._is_atomic: diff --git a/src/etc/lldb_lookup.py b/src/etc/lldb_lookup.py index 94ed47af1891f..8589c0c047ffa 100644 --- a/src/etc/lldb_lookup.py +++ b/src/etc/lldb_lookup.py @@ -333,7 +333,7 @@ def register_providers_compatibility(): # Arc register( - arc_synthetic, + StdRcSyntheticProvider, StdRcSummaryProvider, r"^(alloc::([a-z_]+::)+)Arc<.+>$", ) @@ -483,10 +483,6 @@ def classify_hashset(valobj: lldb.SBValue, _dict: LLDBOpaque) -> object: return StdOldHashMapSyntheticProvider(hash_map, _dict, show_values=False) -def arc_synthetic(valobj: lldb.SBValue, _dict: LLDBOpaque) -> object: - return StdRcSyntheticProvider(valobj, _dict, is_atomic=True) - - def is_udt(type: lldb.SBType, _dict: LLDBOpaque) -> bool: return ( type.GetBasicType() == lldb.eBasicTypeInvalid diff --git a/src/etc/lldb_providers.py b/src/etc/lldb_providers.py index c5c28ac7ad2e5..c54bc2d05b05c 100644 --- a/src/etc/lldb_providers.py +++ b/src/etc/lldb_providers.py @@ -1594,62 +1594,27 @@ class StdRcSyntheticProvider: struct NonZero(T) struct Header { strong: C, weak: C } - struct Arc { ptr: NonNull>, ... } - struct ArcInner { strong: atomic::Atomic, weak: atomic::Atomic, data: T } + struct Arc { ptr: RcValuePointer, ... } """ - def __init__(self, valobj: SBValue, _dict: LLDBOpaque, is_atomic: bool = False): + def __init__(self, valobj: SBValue, _dict: LLDBOpaque): self.valobj = valobj - if is_atomic: - self.ptr = unwrap_unique_or_non_null( - self.valobj.GetChildMemberWithName("ptr") - ) - - self.value = self.ptr.GetChildMemberWithName("data") - - # infallibly gets an unsigned integer type of at least 64 bits. We don't need to worry - # about whether or not `usize` is actually smaller than that since we don't ever display - # the underlying type to the user anyway - usize_type = valobj.GetTarget().GetBasicType(eBasicTypeUnsignedLongLong) - - self.strong = self.ptr.GetChildMemberWithName("strong").Cast(usize_type) - self.weak = self.ptr.GetChildMemberWithName("weak").Cast(usize_type) - - # If the usize type isn't valid due to llvm/llvm-project#196812, not even the type's - # fields will populate. Luckily, `RcInner` is `#[repr(C)]`, so we can infallibly find - # the strong and weak values in memory - if not self.strong.IsValid() or not self.weak.IsValid(): - raw_ptr = self.ptr.Cast(usize_type.GetPointerType()) - addr = raw_ptr.GetValueAsAddress() - - self.strong = self.valobj.CreateValueFromAddress( - "strong", addr, usize_type - ) - - self.weak = self.valobj.CreateValueFromAddress( - "weak", addr + usize_type.GetByteSize(), usize_type - ) - else: - ptr = ( - self.valobj.GetChildMemberWithName("ptr") - .GetChildMemberWithName("ptr") - .GetChildMemberWithName("pointer") - ) - - self.value = ptr.deref.Clone("value") - - target = ptr.GetTarget() - weak_address = ptr.GetValueAsUnsigned() - target.addr_size - strong_address = weak_address - target.addr_size - usize_type = _get_usize_type(target) + ptr = ( + self.valobj.GetChildMemberWithName("ptr") + .GetChildMemberWithName("ptr") + .GetChildMemberWithName("pointer") + ) - self.strong = ptr.CreateValueFromAddress( - "strong", strong_address, usize_type - ) + self.value = ptr.deref.Clone("value") - self.weak = ptr.CreateValueFromAddress("weak", weak_address, usize_type) + target = ptr.GetTarget() + weak_address = ptr.GetValueAsUnsigned() - target.addr_size + strong_address = weak_address - target.addr_size + usize_type = _get_usize_type(target) + self.strong = ptr.CreateValueFromAddress("strong", strong_address, usize_type) + self.weak = ptr.CreateValueFromAddress("weak", weak_address, usize_type) self.value_builder = ValueBuilder(valobj) self.update() diff --git a/src/etc/natvis/liballoc.natvis b/src/etc/natvis/liballoc.natvis index 2b6af726ab578..0c869cc48c8e5 100644 --- a/src/etc/natvis/liballoc.natvis +++ b/src/etc/natvis/liballoc.natvis @@ -131,58 +131,58 @@ - {ptr.pointer->data} + {*ptr.ptr.pointer} - ptr.pointer->data - ptr.pointer->strong - ptr.pointer->weak + *ptr.ptr.pointer + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer - 1)->strong + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer - 1)->weak - ptr.pointer.pointer->strong - ptr.pointer.pointer->weak + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.pointer - 1)->strong + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.pointer - 1)->weak - {{ len={ptr.pointer.length} }} + {{ len={ptr.ptr.pointer.length} }} - ptr.pointer.length - ptr.pointer.data_ptr->strong - ptr.pointer.data_ptr->weak + ptr.ptr.pointer.length + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.data_ptr - 1)->strong + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.data_ptr - 1)->weak - ptr.pointer.length - ($T1*)(((size_t*)ptr.pointer.data_ptr) + 2) + ptr.ptr.pointer.length + ($T1*)ptr.ptr.pointer.data_ptr - {ptr.pointer->data} + {*ptr.ptr.pointer} - ptr.pointer->data - ptr.pointer->strong - ptr.pointer->weak + *ptr.ptr.pointer + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer - 1)->strong + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer - 1)->weak - ptr.pointer.pointer->strong - ptr.pointer.pointer->weak + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.pointer - 1)->strong + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.pointer - 1)->weak - {{ len={ptr.pointer.length} }} + {{ len={ptr.ptr.pointer.length} }} - ptr.pointer.length - ptr.pointer.data_ptr->strong - ptr.pointer.data_ptr->weak + ptr.ptr.pointer.length + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.data_ptr - 1)->strong + ((alloc::rcs::common::Header<core::sync::atomic::Atomic<usize>> *)ptr.ptr.pointer.data_ptr - 1)->weak - ptr.pointer.length - ($T1*)(((size_t*)ptr.pointer.data_ptr) + 2) + ptr.ptr.pointer.length + ($T1*)ptr.ptr.pointer.data_ptr diff --git a/tests/codegen-llvm/issues/issue-111603.rs b/tests/codegen-llvm/issues/issue-111603.rs index 9602233f5b4e1..db7b5de6030b5 100644 --- a/tests/codegen-llvm/issues/issue-111603.rs +++ b/tests/codegen-llvm/issues/issue-111603.rs @@ -20,8 +20,8 @@ pub fn new_from_array(x: u64) -> Arc<[u64]> { // CHECK-LABEL: @new_uninit #[no_mangle] pub fn new_uninit(x: u64) -> Arc<[u64; 1000]> { - // CHECK: call alloc::rcs::arc::arcinner_layout_for_value_layout - // CHECK-NOT: call alloc::rcs::arc::arcinner_layout_for_value_layout + // CHECK: call {{.*}}__rust_alloc(i[[#BITS:]] + // CHECK-SAME: [[#8000 + div(BITS,4)]], i[[#BITS]]{{.*}} 8) # let mut arc = Arc::new_uninit(); unsafe { Arc::get_mut_unchecked(&mut arc) }.write([x; 1000]); unsafe { arc.assume_init() } @@ -30,8 +30,8 @@ pub fn new_uninit(x: u64) -> Arc<[u64; 1000]> { // CHECK-LABEL: @new_uninit_slice #[no_mangle] pub fn new_uninit_slice(x: u64) -> Arc<[u64]> { - // CHECK: call alloc::rcs::arc::arcinner_layout_for_value_layout - // CHECK-NOT: call alloc::rcs::arc::arcinner_layout_for_value_layout + // CHECK: call {{.*}}__rust_alloc(i[[#BITS:]] + // CHECK-SAME: [[#8000 + div(BITS,4)]], i[[#BITS]]{{.*}} 8) # let mut arc = Arc::new_uninit_slice(1000); for elem in unsafe { Arc::get_mut_unchecked(&mut arc) } { elem.write(x); diff --git a/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs b/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs index 9cfb91791a02f..8c8986f907fed 100644 --- a/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs +++ b/tests/codegen-llvm/lib-optimizations/rc-arc-optimizations.rs @@ -3,6 +3,7 @@ #![crate_type = "lib"] use std::rc::{self, Rc}; +use std::sync::{self, Arc}; // Ensures that we can create array of `Weak`s using `memset`. @@ -15,6 +16,15 @@ pub fn array_of_rc_weak() -> [rc::Weak; 100] { [(); 100].map(|()| rc::Weak::new()) } +#[no_mangle] +pub fn array_of_sync_weak() -> [sync::Weak; 100] { + // CHECK-LABEL: @array_of_sync_weak( + // CHECK-NEXT: start: + // CHECK-NEXT: call void @llvm.memset. + // CHECK-NEXT: ret void + [(); 100].map(|()| sync::Weak::new()) +} + // Ensures that we convert `&Option>` and `&Option>` to `Option<&T>` without checking // for `None`. @@ -26,3 +36,12 @@ pub fn option_rc_as_deref_no_cmp(rc: &Option>) -> Option<&u32> { // CHECK-NEXT: ret ptr %[[RC]] rc.as_deref() } + +#[no_mangle] +pub fn option_arc_as_deref_no_cmp(arc: &Option>) -> Option<&u32> { + // CHECK-LABEL: @option_arc_as_deref_no_cmp(ptr + // CHECK-NEXT: start: + // CHECK-NEXT: %[[ARC:.+]] = load ptr, ptr %arc + // CHECK-NEXT: ret ptr %[[ARC]] + arc.as_deref() +} diff --git a/tests/codegen-llvm/placement-new.rs b/tests/codegen-llvm/placement-new.rs index 5022af8f52f19..28cc782476980 100644 --- a/tests/codegen-llvm/placement-new.rs +++ b/tests/codegen-llvm/placement-new.rs @@ -34,8 +34,10 @@ pub fn rc_default_inplace() -> Rc<(String, String)> { #[no_mangle] pub fn arc_default_inplace() -> Arc<(String, String)> { // CHECK-NOT: alloca - // CHECK: [[ARC:%.*]] = {{.*}}call {{.*}}__rust_alloc( + // CHECK: [[ARC:%.*]] = {{.*}}call {{.*}}__rust_alloc(i[[#BITS:]] // CHECK-NOT: call void @llvm.memcpy - // CHECK: ret ptr [[ARC]] + // CHECK: [[DATA:%.*]] = getelementptr inbounds{{( nuw)?}} i8, ptr [[ARC]], i[[#BITS]] [[#div(BITS,4)]] + // CHECK-NOT: call void @llvm.memcpy + // CHECK: ret ptr [[DATA]] Arc::default() } diff --git a/tests/debuginfo/rc_arc.rs b/tests/debuginfo/rc_arc.rs index adeb96dba1b80..17a2b0c414412 100644 --- a/tests/debuginfo/rc_arc.rs +++ b/tests/debuginfo/rc_arc.rs @@ -20,7 +20,7 @@ //@ lldb-command:v rc //@ lldb-check:[...] strong=11, weak=2 { value = 111 } //@ lldb-command:v arc -//@ lldb-check:[...] strong=21, weak=2 { data = 222 } +//@ lldb-check:[...] strong=21, weak=2 { value = 222 } // === CDB TESTS ================================================================================== From ab74ae3579f1509eb4a638121ee1cb883698dc1e Mon Sep 17 00:00:00 2001 From: EFanZh Date: Sun, 11 Oct 2026 23:04:38 +0800 Subject: [PATCH 4/4] Try some optimizations --- library/alloc/src/rcs/arc.rs | 285 ++++++++++++++++++++--------------- 1 file changed, 166 insertions(+), 119 deletions(-) diff --git a/library/alloc/src/rcs/arc.rs b/library/alloc/src/rcs/arc.rs index 018f552219324..c1d8990e0350d 100644 --- a/library/alloc/src/rcs/arc.rs +++ b/library/alloc/src/rcs/arc.rs @@ -23,7 +23,7 @@ use crate::alloc::{ use crate::borrow::{Cow, ToOwned}; #[cfg(not(no_global_oom_handling))] use crate::boxed::Box; -use crate::rcs::common::{ErasedRcValuePointer, Header, RcValuePointer}; +use crate::rcs::common::{ErasedRcValuePointer, Header, HeaderRef, MaybeHeaderRef, RcValuePointer}; #[cfg(not(no_global_oom_handling))] use crate::string::String; #[cfg(not(no_global_oom_handling))] @@ -312,10 +312,7 @@ impl Arc { unsafe fn init_from_parts(ptr: RcValuePointer, alloc: A) -> Self { // SAFETY: Upheld by caller. unsafe { - // `ArcHeader::INIT_ARC` is no used here because that generates `memcpy` calls that fails - // `tests/codegen-llvm/placement-new.rs` test. - ArcHeader { weak: Atomic::::new(1), strong: Atomic::::new(1) } - .write_to(ptr.erase()); + write_arc_header(ptr.erase()); Self::from_inner_in(ptr, alloc) } @@ -388,17 +385,36 @@ const _: () = { assert!(size_of::() - mem::offset_of!(ArcHeader, weak) == size_of::()); }; -#[expect(clippy::declare_interior_mutable_const, reason = "by design")] -impl ArcHeader { - /// Initial header for newly created `Arc` value. The initial weak count of 1 used for tracking - /// all strong counts. - const INIT_ARC: Self = Self { weak: Atomic::::new(1), strong: Atomic::::new(1) }; +/// Writes initial header for newly created `Arc` value. The initial weak count of 1 used for tracking +/// all strong counts. +/// +/// # Safety +/// +/// `ptr` must point to a valid reference counted allocation with mutable access. +#[inline] +unsafe fn write_arc_header(ptr: ErasedRcValuePointer) { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { + ArcHeader { strong: Atomic::::new(1), weak: Atomic::::new(1) }.write_to(ptr); + } +} - /// Initial header for newly created `UniqueRc` value. - const INIT_UNIQUE_ARC: Self = - Self { weak: Atomic::::new(1), strong: Atomic::::new(0) }; +/// Writes initial header for newly created `UniqueArc` value. +/// +/// # Safety +/// +/// `ptr` must point to a valid reference counted allocation with mutable access. +#[inline] +unsafe fn write_unique_arc_header(ptr: ErasedRcValuePointer) { + // SAFETY: Caller guarantees the validity of `ptr`. + unsafe { + ArcHeader { strong: Atomic::::new(0), weak: Atomic::::new(1) }.write_to(ptr); + } } +type ArcHeaderRef<'a> = HeaderRef<'a, Atomic>; +type MaybeArcHeaderRef<'a> = MaybeHeaderRef<'a, Atomic>; + impl Arc { /// Constructs a new `Arc`. /// @@ -627,7 +643,7 @@ impl Arc { /// /// let five = Arc::new_in(5, System); /// ``` - #[inline] + #[inline(always)] #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] pub fn new_in(data: T, alloc: A) -> Arc { @@ -748,7 +764,10 @@ impl Arc { let init_ptr = ArcHeader::allocate_uninit_for_type(&alloc); // SAFETY: We have exclusive access to `init_ptr`. - unsafe { ArcHeader::INIT_UNIQUE_ARC.write_to(init_ptr.erase()) }; + unsafe { write_unique_arc_header(init_ptr.erase()) }; + + // SAFETY: `init_ptr` is guaranteed to be valid. + unsafe { hint::assert_unchecked(!init_ptr.is_dangling()) }; let weak = Weak { ptr: init_ptr, alloc }; @@ -776,7 +795,7 @@ impl Arc { // // These side effects do not impact us in any way, and no other side effects are // possible with safe code alone. - let prev_value = ArcHeader::get(init_ptr.erase()).strong.fetch_add(1, Release); + let prev_value = ArcHeaderRef::new(init_ptr.erase()).strong.fetch_add(1, Release); debug_assert_eq!(prev_value, 0, "No prior strong references should exist"); // Strong references should collectively own a shared weak reference, @@ -956,7 +975,7 @@ impl Arc { let alloc: A = unsafe { ptr::read(&this.alloc) }; // copy the allocator // Make a weak pointer to clean up the implicit strong-weak reference - let _weak = Weak { ptr: this.ptr, alloc }; + let _weak = Weak { ptr: this.non_dangling_ptr(), alloc: &alloc }; Ok(elem) } @@ -1081,7 +1100,7 @@ impl Arc { // SAFETY: Pointer is valid for reads. let alloc = unsafe { ptr::read(&this.alloc) }; - drop(Weak { ptr: this.ptr, alloc }); + drop(Weak { ptr: this.non_dangling_ptr(), alloc: &alloc }); Some(inner) } @@ -1725,7 +1744,9 @@ impl Arc { // SAFETY: Caller guarantees `ptr` points to a valid reference counted allocation, we are // safe to get a reference to the header. let header = unsafe { - ArcHeader::get(ErasedRcValuePointer::new(NonNull::new_unchecked(ptr.cast_mut().cast()))) + ArcHeaderRef::new(ErasedRcValuePointer::new(NonNull::new_unchecked( + ptr.cast_mut().cast(), + ))) }; header.strong.load(Relaxed) @@ -1895,38 +1916,42 @@ impl Arc { where A: AllocatorClone, { - // This Relaxed is OK because we're checking the value in the CAS - // below. - let mut cur = this.inner().weak.load(Relaxed); - - loop { - // check if the weak counter is currently "locked"; if so, spin. - if cur == usize::MAX { - hint::spin_loop(); - cur = this.inner().weak.load(Relaxed); - continue; - } + #[inline] + fn inc_weak(header_ref: ArcHeaderRef<'_>) { + // This Relaxed is OK because we're checking the value in the CAS + // below. + let mut cur = header_ref.weak.load(Relaxed); + + loop { + // check if the weak counter is currently "locked"; if so, spin. + if cur == usize::MAX { + hint::spin_loop(); + cur = header_ref.weak.load(Relaxed); + continue; + } - // We can't allow the refcount to increase much past `MAX_REFCOUNT`. - if cur > MAX_REFCOUNT { - panic_arc_overflow(); - } - // NOTE: this code currently ignores the possibility of overflow - // into usize::MAX; in general both Rc and Arc need to be adjusted - // to deal with overflow. - - // Unlike with Clone(), we need this to be an Acquire read to - // synchronize with the write coming from `is_unique`, so that the - // events prior to that write happen before this read. - match this.inner().weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) { - Ok(_) => { - // Make sure we do not create a dangling Weak - debug_assert!(!this.ptr.is_dangling()); - return Weak { ptr: this.ptr, alloc: this.alloc.clone() }; + // We can't allow the refcount to increase much past `MAX_REFCOUNT`. + if cur > MAX_REFCOUNT { + panic_arc_overflow(); + } + // NOTE: this code currently ignores the possibility of overflow + // into usize::MAX; in general both Rc and Arc need to be adjusted + // to deal with overflow. + + // Unlike with Clone(), we need this to be an Acquire read to + // synchronize with the write coming from `is_unique`, so that the + // events prior to that write happen before this read. + match header_ref.weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) { + Ok(_) => return, + Err(old) => cur = old, } - Err(old) => cur = old, } } + + inc_weak(this.inner()); + + // Make sure we do not create a dangling Weak + return Weak { ptr: this.non_dangling_ptr(), alloc: this.alloc.clone() }; } /// Gets the number of [`Weak`] pointers to this allocation. @@ -2078,22 +2103,36 @@ impl Arc { unsafe { drop(Arc::from_raw_in(ptr, alloc)) }; } - #[inline] - fn inner(&self) -> &ArcHeader { + #[inline(always)] + fn inner(&self) -> ArcHeaderRef<'_> { // SAFETY: While this arc is alive we're guaranteed // that the header is valid. - unsafe { ArcHeader::get(self.ptr.erase()) } + unsafe { ArcHeaderRef::new(self.ptr.erase()) } + } + + #[inline(always)] + fn non_dangling_ptr(&self) -> RcValuePointer { + // SAFETY: `Arc` always contains a non-dangling value pointer. + unsafe { hint::assert_unchecked(!self.ptr.is_dangling()) } + + self.ptr } // Non-inlined part of `drop`. #[inline(never)] unsafe fn drop_slow(&mut self) { + // ignore-tidy-undocumented-unsafe + unsafe { self.drop_slow_inlined() }; + } + + #[inline(always)] + unsafe fn drop_slow_inlined(&mut self) { // Drop the weak ref collectively held by all strong references when this // variable goes out of scope. This ensures that the memory is deallocated // even if the destructor of `T` panics. // Take a reference to `self.alloc` instead of cloning because 1. it'll last long // enough, and 2. you should be able to drop `Arc`s with unclonable allocators - let _weak = Weak { ptr: self.ptr, alloc: &self.alloc }; + let _weak = Weak { ptr: self.non_dangling_ptr(), alloc: &self.alloc }; // Destroy the data at this time, even though we must not free the box // allocation itself (there might still be weak pointers lying around). @@ -2127,7 +2166,8 @@ impl Arc { } } -unsafe fn inc_strong(ptr: ErasedRcValuePointer) { +#[inline] +fn inc_strong(header_ref: ArcHeaderRef<'_>) { // Using a relaxed ordering is alright here, as knowledge of the // original reference prevents other threads from erroneously deleting // the object. @@ -2141,7 +2181,7 @@ unsafe fn inc_strong(ptr: ErasedRcValuePointer) { // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) // SAFETY: Caller guarantees the validity of `ptr`. - let old_size = unsafe { ArcHeader::get(ptr) }.strong.fetch_add(1, Relaxed); + let old_size = header_ref.strong.fetch_add(1, Relaxed); // However we need to guard against massive refcounts in case someone is `mem::forget`ing // Arcs. If we don't do this the count can overflow and users will use-after free. This @@ -2181,12 +2221,10 @@ impl Clone for Arc { /// ``` #[inline] fn clone(&self) -> Arc { - // SAFETY: Pointer is valid & allocator corresponds to the one used to allocate it. - unsafe { - inc_strong(self.ptr.erase()); + inc_strong(self.inner()); - Self::from_inner_in(self.ptr, self.alloc.clone()) - } + // SAFETY: Pointer is valid & allocator corresponds to the one used to allocate it. + unsafe { Self::from_inner_in(self.ptr, self.alloc.clone()) } } } @@ -2310,7 +2348,7 @@ impl Arc { // count of 0, resulting in a UAF (#155746, #157203). // Until the new Arc is written, the old Arc must remain valid struct Guard<'a> { - inner: &'a ArcHeader, + inner: ArcHeaderRef<'a>, } impl<'a> Drop for Guard<'a> { fn drop(&mut self) { @@ -2344,7 +2382,7 @@ impl Arc { // up the allocation as needed. // Make sure the allocator is not leaked when the Arc is overwritten. // Only drop at the end of the scope to avoid panics. - let _weak = Weak { ptr: this.ptr, alloc: ptr::read(&this.alloc) }; + let _weak = Weak { ptr: this.non_dangling_ptr(), alloc: &ptr::read(&this.alloc) }; ptr::write(this, in_progress.into_arc()); } @@ -2663,7 +2701,11 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Arc { // ignore-tidy-undocumented-unsafe unsafe { - self.drop_slow(); + if const { mem::needs_drop::() } { + self.drop_slow(); + } else { + self.drop_slow_inlined(); + } } } } @@ -3061,7 +3103,7 @@ impl Weak { // Acquire is necessary for the success case to synchronise with `Arc::new_cyclic`, when the inner // value can be initialized after `Weak` references have already been created. In that case, we // expect to observe the fully initialized value. - if self.inner()?.strong.try_update(Acquire, Relaxed, checked_increment).is_ok() { + if self.inner().header()?.strong.try_update(Acquire, Relaxed, checked_increment).is_ok() { // SAFETY: pointer is not null, verified in checked_increment unsafe { Some(Arc::from_inner_in(self.ptr, self.alloc.clone())) } } else { @@ -3075,7 +3117,7 @@ impl Weak { #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn strong_count(&self) -> usize { - if let Some(inner) = self.inner() { inner.strong.load(Relaxed) } else { 0 } + if let Some(inner) = self.inner().header() { inner.strong.load(Relaxed) } else { 0 } } /// Gets an approximation of the number of `Weak` pointers pointing to this @@ -3092,7 +3134,7 @@ impl Weak { #[must_use] #[stable(feature = "weak_counts", since = "1.41.0")] pub fn weak_count(&self) -> usize { - if let Some(inner) = self.inner() { + if let Some(inner) = self.inner().header() { let weak = inner.weak.load(Acquire); let strong = inner.strong.load(Relaxed); if strong == 0 { @@ -3110,21 +3152,11 @@ impl Weak { } } - /// Returns `None` when the pointer is dangling and there is no allocated value, - /// (i.e., when this `Weak` was created by `Weak::new`). - #[inline] - fn inner(&self) -> Option<&ArcHeader> { - let ptr = self.ptr; - - if ptr.is_dangling() { - None - } else { - // We are careful to *not* create a reference covering the "data" field, as - // the field may be mutated concurrently (for example, if the last `Arc` - // is dropped, the data field will be dropped in-place). - // ignore-tidy-undocumented-unsafe - Some(unsafe { ArcHeader::get(ptr.erase()) }) - } + /// Returns an accessor to the reference counters. + #[inline(always)] + fn inner(&self) -> MaybeArcHeaderRef<'_> { + // SAFETY: `self.ptr` is either valid or danging. + unsafe { MaybeArcHeaderRef::new(self.ptr.erase()) } } /// Returns `true` if the two `Weak`s point to the same allocation similar to [`ptr::eq`], or if @@ -3191,19 +3223,24 @@ impl Clone for Weak { /// ``` #[inline] fn clone(&self) -> Weak { - if let Some(inner) = self.inner() { - // See comments in Arc::clone() for why this is relaxed. This can use a - // fetch_add (ignoring the lock) because the weak count is only locked - // where are *no other* weak pointers in existence. (So we can't be - // running this code in that case). - let old_size = inner.weak.fetch_add(1, Relaxed); - - // See comments in Arc::clone() for why we do this (for mem::forget). - if old_size > MAX_REFCOUNT { - intrinsics::abort_immediate(); + #[inline] + fn inc_weak(header: MaybeArcHeaderRef<'_>) { + if let Some(inner) = header.header() { + // See comments in Arc::clone() for why this is relaxed. This can use a + // fetch_add (ignoring the lock) because the weak count is only locked + // where are *no other* weak pointers in existence. (So we can't be + // running this code in that case). + let old_size = inner.weak.fetch_add(1, Relaxed); + + // See comments in Arc::clone() for why we do this (for mem::forget). + if old_size > MAX_REFCOUNT { + intrinsics::abort_immediate(); + } } } + inc_weak(self.inner()); + Weak { ptr: self.ptr, alloc: self.alloc.clone() } } } @@ -3267,7 +3304,7 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Weak { // weak count can only be locked if there was precisely one weak ref, // meaning that drop could only subsequently run ON that remaining weak // ref, which can only happen after the lock is released. - let inner = if let Some(inner) = self.inner() { inner } else { return }; + let Some(inner) = self.inner().header() else { return }; if inner.weak.fetch_sub(1, Release) == 1 { acquire!(inner.weak); @@ -3546,7 +3583,7 @@ const MAX_STATIC_INNER_SLICE_ALIGNMENT: usize = 16; static STATIC_INNER_SLICE: SliceArcInnerForStatic = SliceArcInnerForStatic { _padding: MaybeUninit::uninit(), - header: ArcHeader::INIT_ARC, + header: ArcHeader { strong: Atomic::::new(1), weak: Atomic::::new(1) }, value: [0], }; @@ -3575,7 +3612,7 @@ where // SAFETY: `SliceArcInnerForStatic` is designed to hold a valid reference counted // allocation, and caller guarantees that the value can be shared as `T`. unsafe { - inc_strong(STATIC_INNER_SLICE_VALUE_PTR); + inc_strong(ArcHeaderRef::new(STATIC_INNER_SLICE_VALUE_PTR)); Arc::from_inner_in(STATIC_INNER_SLICE_VALUE_PTR.unerase_with(metadata), alloc) } @@ -3980,6 +4017,7 @@ struct UniqueArcUninit { impl UniqueArcUninit { /// Allocates a reference counted allocation with layout suitable to contain `for_value` or a /// clone of it. + #[inline(always)] fn new(arc: &Arc, alloc: A) -> UniqueArcUninit { let ptr = // SAFETY: `Arc` proves that `T` is small enough to store inside a reference @@ -3988,7 +4026,7 @@ impl UniqueArcUninit { // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive // access to it. - unsafe { ArcHeader::INIT_ARC.write_to(ptr.erase()) }; + unsafe { write_arc_header(ptr.erase()) }; Self { ptr, alloc } } @@ -4362,7 +4400,7 @@ impl UniqueArc { // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive // access to it. unsafe { - ArcHeader::INIT_UNIQUE_ARC.write_to(ptr.erase()); + write_unique_arc_header(ptr.erase()); ptr.ptr.write(data); } @@ -4378,7 +4416,7 @@ impl UniqueArc { // SAFETY: `ptr` points to a newly allocated reference counted allocation, we have exclusive // access to it. unsafe { - ArcHeader::INIT_UNIQUE_ARC.write_to(ptr.erase()); + write_unique_arc_header(ptr.erase()); ptr.ptr.write(data); } @@ -4391,7 +4429,7 @@ impl UniqueArc { // #[unstable(feature = "allocator_api", issue = "163177")] #[must_use] pub fn unwrap_with_allocator(this: Self) -> (T, A) { - let inner_ptr = this.ptr; + let inner_ptr = this.non_dangling_ptr(); let (data_ptr, alloc) = Self::into_raw_with_allocator(this); // SAFETY: Conceptually moves out of the `UniqueRc`. @@ -4541,12 +4579,10 @@ impl UniqueArc { // a `ManuallyDrop`. let alloc: A = unsafe { ptr::read(&this.alloc) }; + this.inner().strong.store(1, Release); + // SAFETY: This pointer was allocated at creation time so we know it is valid. - unsafe { - // Convert our weak reference into a strong reference - ArcHeader::get(this.ptr.erase()).strong.store(1, Release); - Arc::from_inner_in(this.ptr, alloc) - } + unsafe { Arc::from_inner_in(this.ptr, alloc) } } #[cfg(not(no_global_oom_handling))] @@ -4554,10 +4590,18 @@ impl UniqueArc { this.inner().weak.load(Acquire) - 1 } - #[cfg(not(no_global_oom_handling))] - fn inner(&self) -> &ArcHeader { + #[inline(always)] + fn inner(&self) -> ArcHeaderRef<'_> { // SAFETY: while this UniqueArc is alive we're guaranteed that the inner pointer is valid. - unsafe { ArcHeader::get(self.ptr.erase()) } + unsafe { ArcHeaderRef::new(self.ptr.erase()) } + } + + #[inline(always)] + fn non_dangling_ptr(&self) -> RcValuePointer { + // SAFETY: `UniqueArc` always contains a non-dangling value pointer. + unsafe { hint::assert_unchecked(!self.ptr.is_dangling()) } + + self.ptr } fn as_ptr(this: &Self) -> *const T { @@ -4585,23 +4629,26 @@ impl UniqueArc { #[unstable(feature = "unique_rc_arc", issue = "112566")] #[must_use] pub fn downgrade(this: &Self) -> Weak { - // Using a relaxed ordering is alright here, as knowledge of the - // original reference prevents other threads from erroneously deleting - // the object or converting the object to a normal `Arc`. - // - // Note that we don't need to test if the weak counter is locked because there - // are no such operations like `Arc::get_mut` or `Arc::make_mut` that will lock - // the weak counter. - // - // SAFETY: This pointer was allocated at creation time so we know it is valid. - let old_size = unsafe { ArcHeader::get(this.ptr.erase()).weak.fetch_add(1, Relaxed) }; + #[inline] + fn inc_weak(header_ref: ArcHeaderRef<'_>) { + // Using a relaxed ordering is alright here, as knowledge of the + // original reference prevents other threads from erroneously deleting + // the object or converting the object to a normal `Arc`. + // + // Note that we don't need to test if the weak counter is locked because there + // are no such operations like `Arc::get_mut` or `Arc::make_mut` that will lock + // the weak counter. + let old_size = header_ref.weak.fetch_add(1, Relaxed); - // See comments in Arc::clone() for why we do this (for mem::forget). - if old_size > MAX_REFCOUNT { - intrinsics::abort_immediate(); + // See comments in Arc::clone() for why we do this (for mem::forget). + if old_size > MAX_REFCOUNT { + intrinsics::abort_immediate(); + } } - Weak { ptr: this.ptr, alloc: this.alloc.clone() } + inc_weak(this.inner()); + + Weak { ptr: this.non_dangling_ptr(), alloc: this.alloc.clone() } } } @@ -4676,7 +4723,7 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for UniqueArc { fn drop(&mut self) { // See `Arc::drop_slow` which drops an `Arc` with a strong count of 0. // SAFETY: This pointer was allocated at creation time so we know it is valid. - let _weak = Weak { ptr: self.ptr, alloc: &self.alloc }; + let _weak = Weak { ptr: self.non_dangling_ptr(), alloc: &self.alloc }; // ignore-tidy-undocumented-unsafe unsafe { self.ptr.ptr.drop_in_place() };