diff --git a/majit/majit-backend-cranelift/Cargo.toml b/majit/majit-backend-cranelift/Cargo.toml index 4ba2adec2f7..1a4dff7ec73 100644 --- a/majit/majit-backend-cranelift/Cargo.toml +++ b/majit/majit-backend-cranelift/Cargo.toml @@ -33,6 +33,7 @@ dynasm = ["majit-metainterp/dynasm"] cranelift = ["majit-metainterp/cranelift"] [dev-dependencies] +majit-gc = { workspace = true, features = ["gc_box"] } majit-ir = { workspace = true, features = ["test-support"] } majit-metainterp = { workspace = true } smallvec = { workspace = true } diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index 4a34cf5d512..45863f70edd 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -1417,32 +1417,141 @@ fn cranelift_type_for(tp: &Type) -> cranelift_codegen::ir::Type { } thread_local! { - /// `llmodel.py:58` `self.gc_ll_descr = get_ll_description(...)` — owned - /// by the active cranelift backend on this thread. Stored as a - /// thread-local so the backend-agnostic `majit_gc::ActiveGcGuardHooks` - /// shims and trampoline-baked C addresses can reach the live allocator - /// without taking a cranelift dependency. Mirrors - /// `majit-backend-dynasm/src/runner.rs:46 DYNASM_ACTIVE_GC`. - static CRANELIFT_ACTIVE_GC: RefCell>> = - const { RefCell::new(None) }; - static CRANELIFT_ACTIVE_GC_RAW: Cell> = - const { Cell::new(None) }; /// JITFRAME `Type` id of the active GC runtime. Lazy-registered when /// the GC sees its first JITFRAME, cleared when the active GC is /// replaced or torn down. static CRANELIFT_JITFRAME_TYPE_ID: Cell> = const { Cell::new(None) }; } -fn with_cranelift_gc(f: impl FnOnce(&mut dyn GcAllocator) -> R) -> Option { - if majit_gc::gc_box_installed() && CRANELIFT_ACTIVE_GC.with(|cell| cell.borrow().is_some()) { - return CRANELIFT_ACTIVE_GC.with(|cell| { +/// The per-thread GC box, and the accessors every trampoline reaches it through. +/// +/// `gc.py:30` `GcLLDescription.__init__` holds `self.gcdescr` as a plain field +/// on the backend descriptor — there is no per-thread allocator upstream — so +/// this cell is scaffolding, not a ported structure. Only `install_gc_box` +/// fills it and only tests reach that; the production build goes through +/// `install_gc_standalone` and allocates from the `gc_sync` singleton. +/// +/// Every accessor opens with `majit_gc::gc_box_installed()`, which without +/// `majit-gc/gc_box` is a constant `false` — so in a production build each one +/// folds to `None`, the thread-local becomes unreachable, and the trampolines +/// call `gc_sync` directly. The gate lives in `majit-gc` because a Cargo +/// feature is per-crate: this crate cannot `#[cfg]` on a feature of its +/// dependency, so the box is eliminated by the optimizer rather than by +/// conditional compilation. Mirrors `majit-backend-dynasm/src/runner.rs`'s +/// `gc_box`. +mod gc_box { + use super::{Cell, GcAllocator, RefCell}; + + thread_local! { + /// `llmodel.py:58` `self.gc_ll_descr = get_ll_description(...)` — owned + /// by the active cranelift backend on this thread. Stored as a + /// thread-local so the backend-agnostic `majit_gc::ActiveGcGuardHooks` + /// shims and trampoline-baked C addresses can reach the live allocator + /// without taking a cranelift dependency. + static CRANELIFT_ACTIVE_GC: RefCell>> = + const { RefCell::new(None) }; + /// Read-only mirror of the box address, for the queries that can fire + /// while an in-progress allocation already holds the mutable borrow. + static CRANELIFT_ACTIVE_GC_RAW: Cell> = + const { Cell::new(None) }; + } + + /// `&mut` access to this thread's GC box, for allocation and write + /// barriers. `None` means there is no box, and the caller runs its + /// `gc_sync` path instead. + pub(super) fn with_mut(f: impl FnOnce(&mut dyn GcAllocator) -> R) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + CRANELIFT_ACTIVE_GC.with(|cell| { let mut guard = cell.borrow_mut(); let raw: *mut dyn GcAllocator = guard.as_deref_mut()?; // SAFETY: `guard` holds the borrow for the whole `f` call and - // these are non-reentrant top-level trampolines, so the - // reborrow is exclusive and outlives `f`. + // these are non-reentrant top-level trampolines, so the reborrow + // is exclusive and outlives `f`. Some(f(unsafe { &mut *raw })) + }) + } + + /// Read-only access that tolerates being reached from inside a collection. + /// + /// Structural adaptation: RPython's GC descriptor is a normal object + /// reference, so `gc_current_object_address` can query ownership while a + /// collection is already walking extra roots. Here the box sits behind a + /// `RefCell` whose mutable borrow an allocation slowpath may hold, so that + /// case reads the same allocator through the raw mirror rather than + /// panicking across the extern slowpath. + pub(super) fn with_reentrant_ref(f: impl FnOnce(&dyn GcAllocator) -> R) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + CRANELIFT_ACTIVE_GC.with(|cell| match cell.try_borrow() { + Ok(guard) => guard.as_deref().map(f), + // SAFETY: the mirror is published and cleared under the same + // borrow as the box itself, so a non-null value points at the + // live allocator, and this query only reads it. + Err(_) => CRANELIFT_ACTIVE_GC_RAW.with(|raw| raw.get().map(|p| f(unsafe { &*p }))), + }) + } + + /// `&mut` access for a top-level, never-reentrant op that has a defined + /// answer when the box is busy: a borrow already held by an in-progress + /// allocation yields `busy` rather than falling through to `gc_sync`. + pub(super) fn with_mut_or_busy( + busy: R, + f: impl FnOnce(&mut dyn GcAllocator) -> R, + ) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + CRANELIFT_ACTIVE_GC.with(|cell| { + let mut guard = match cell.try_borrow_mut() { + Ok(guard) => guard, + Err(_) => return Some(busy), + }; + let raw: *mut dyn GcAllocator = guard.as_deref_mut()?; + // SAFETY: as in [`with_mut`]. + Some(f(unsafe { &mut *raw })) + }) + } + + /// Whether this thread holds a box at all. + pub(super) fn present() -> bool { + majit_gc::gc_box_installed() && CRANELIFT_ACTIVE_GC.with(|cell| cell.borrow().is_some()) + } + + /// Install or drop this thread's box, keeping the raw mirror in step. + pub(super) fn store(gc: Option>) { + CRANELIFT_ACTIVE_GC.with(|cell| { + let mut guard = cell.borrow_mut(); + // Drop the previous allocator first so reentrant + // `is_managed_heap_object` queries from its drop body still + // resolve old-heap addresses through the raw mirror — it keeps + // pointing at the live old box throughout the drop body. + // Publishing the new raw pointer before the drop would route + // those queries to the new allocator, which does not know about + // old-heap addresses and would report them as unmanaged. After + // the drop returns no further reentry is possible on this thread + // before the raw mirror is republished synchronously below. + *guard = gc; + let raw = guard.as_deref_mut().map(|gc| gc as *mut dyn GcAllocator); + CRANELIFT_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(raw)); + }); + } + + /// Backend-teardown counterpart of [`store`], tolerant of a thread whose + /// thread-locals are already being destroyed. + pub(super) fn clear_on_teardown() { + let _ = CRANELIFT_ACTIVE_GC.try_with(|cell| { + *cell.borrow_mut() = None; }); + let _ = CRANELIFT_ACTIVE_GC_RAW.try_with(|raw_cell| raw_cell.set(None)); + } +} + +fn with_cranelift_gc(f: impl FnOnce(&mut dyn GcAllocator) -> R) -> Option { + if gc_box::present() { + return gc_box::with_mut(f); } if majit_gc::gc_sync::is_initialized() { return Some(majit_gc::gc_sync::gc_op(|gc| f(gc))); @@ -1458,22 +1567,7 @@ fn set_cranelift_active_gc(gc: Option>) { if gc.is_some() { majit_gc::note_gc_box_installed(); } - CRANELIFT_ACTIVE_GC.with(|cell| { - let mut guard = cell.borrow_mut(); - // Drop the previous allocator first so reentrant - // `is_managed_heap_object` queries from its drop body still - // resolve old-heap addresses through the raw mirror — it keeps - // pointing at the live old box throughout the drop body. - // Publishing the new raw pointer before the drop would route - // those queries to the new allocator, which does not know - // about old-heap addresses and would report them as - // unmanaged. After the drop returns no further reentry is - // possible on this thread before the raw mirror is republished - // synchronously below. - *guard = gc; - let raw = guard.as_deref_mut().map(|gc| gc as *mut dyn GcAllocator); - CRANELIFT_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(raw)); - }); + gc_box::store(gc); } fn cranelift_jitframe_type_id() -> Option { @@ -1485,8 +1579,7 @@ fn set_cranelift_jitframe_type_id(type_id: Option) { } fn cranelift_gc_active() -> bool { - (majit_gc::gc_box_installed() && CRANELIFT_ACTIVE_GC.with(|cell| cell.borrow().is_some())) - || majit_gc::gc_sync::is_initialized() + gc_box::present() || majit_gc::gc_sync::is_initialized() } /// `majit_gc::CheckIsObjectFn` installed by `set_gc_allocator`. Dispatches @@ -1641,7 +1734,7 @@ fn get_objects_via_active_runtime(generation: i8, visitor: majit_gc::GetObjectsV /// leaving the collector to answer about a forwarding stub. fn get_referents_via_active_runtime(obj: GcRef, visitor: majit_gc::GetObjectsVisitorFn) { let mut visit = visitor; - if majit_gc::gc_box_installed() && CRANELIFT_ACTIVE_GC.with(|cell| cell.borrow().is_some()) { + if gc_box::present() { with_cranelift_gc(|gc| gc.get_referents(obj, &mut visit)); } else if majit_gc::gc_sync::is_initialized() { majit_gc::gc_sync::gc_op_with_root(obj, |gc, obj| gc.get_referents(obj, &mut visit)); @@ -1649,7 +1742,7 @@ fn get_referents_via_active_runtime(obj: GcRef, visitor: majit_gc::GetObjectsVis } fn is_tracked_via_active_runtime(obj: GcRef) -> bool { - if majit_gc::gc_box_installed() && CRANELIFT_ACTIVE_GC.with(|cell| cell.borrow().is_some()) { + if gc_box::present() { return with_cranelift_gc(|gc| gc.is_tracked(obj)).unwrap_or(false); } if majit_gc::gc_sync::is_initialized() { @@ -1708,7 +1801,7 @@ fn gc_remove_root_via_active_runtime(slot: *mut GcRef) { /// Host-side write-barrier trampoline for GC-managed objects updated /// outside compiled code. fn gc_write_barrier_via_active_runtime(obj: GcRef) { - if majit_gc::gc_box_installed() && CRANELIFT_ACTIVE_GC.with(|cell| cell.borrow().is_some()) { + if gc_box::present() { with_cranelift_gc(|gc| gc.write_barrier(obj)); } else if majit_gc::gc_sync::is_initialized() { majit_gc::gc_sync::gc_op_with_root(obj, |gc, obj| gc.write_barrier(obj)); @@ -1720,16 +1813,10 @@ fn gc_write_barrier_via_active_runtime(obj: GcRef) { /// `try_gc_alloc_stable`-allocated blocks from `std::alloc`-backed /// fallback blocks during the L1/L2 stepping-stone window. fn id_or_identityhash_via_active_runtime(addr: usize) -> usize { - let via_box = majit_gc::gc_box_installed().then(|| { - CRANELIFT_ACTIVE_GC.with(|cell| { - let mut guard = match cell.try_borrow_mut() { - Ok(guard) => guard, - Err(_) => return Some(addr), - }; - guard.as_deref_mut().map(|gc| gc.id_or_identityhash(addr)) - }) - }); - if let Some(Some(r)) = via_box { + // A box whose borrow is already held by an in-progress alloc answers with + // the raw `addr`, not with the singleton's id: this is a top-level op, so + // the busy borrow means the box is mid-allocation, not that it is absent. + if let Some(r) = gc_box::with_mut_or_busy(addr, |gc| gc.id_or_identityhash(addr)) { return r; } if majit_gc::gc_sync::is_initialized() { @@ -1739,62 +1826,22 @@ fn id_or_identityhash_via_active_runtime(addr: usize) -> usize { } fn gc_owns_object_via_active_runtime(addr: usize) -> bool { - // Structural adaptation: RPython's GC descriptor can be queried - // reentrantly while a collection is walking extra roots. Pyre stores - // the active cranelift GC in a Rust `RefCell`; allocation slowpaths - // hold the mutable borrow while those root walkers may call - // `gc_current_object_address`. Use the raw pointer / reentrant - // `gc_sync` read only for this immutable ownership query, matching the - // read-only nature of RPython's descriptor call, instead of panicking - // across the extern slowpath. - if !majit_gc::gc_box_installed() { - return majit_gc::gc_sync::is_initialized() - && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)); - } - match CRANELIFT_ACTIVE_GC.with(|cell| { - cell.try_borrow() - .map(|g| g.as_deref().map(|gc| gc.is_managed_heap_object(addr))) - }) { - Ok(Some(r)) => r, - Ok(None) => { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) - } else { - false - } - } - Err(_) => CRANELIFT_ACTIVE_GC_RAW.with(|raw| match raw.get() { - Some(ptr) => unsafe { (&*ptr).is_managed_heap_object(addr) }, - None => { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) - } else { - false - } - } - }), + // This query can fire reentrantly from an extra-root walker mid-collection, + // so both arms are read-only: `with_reentrant_ref` for the box, and the + // reentrant singleton read for everything else. + if let Some(r) = gc_box::with_reentrant_ref(|gc| gc.is_managed_heap_object(addr)) { + return r; } + majit_gc::gc_sync::is_initialized() + && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) } fn gc_is_nursery_object_via_active_runtime(addr: usize) -> bool { - let via_box = majit_gc::gc_box_installed() - .then(|| { - CRANELIFT_ACTIVE_GC.with(|cell| match cell.try_borrow() { - Ok(guard) => guard.as_deref().map(|gc| gc.is_nursery_object(addr)), - Err(_) => CRANELIFT_ACTIVE_GC_RAW.with(|raw| { - raw.get() - .map(|ptr| unsafe { (&*ptr).is_nursery_object(addr) }) - }), - }) - }) - .flatten(); - via_box.unwrap_or_else(|| { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_nursery_object(addr)) - } else { - false - } - }) + if let Some(r) = gc_box::with_reentrant_ref(|gc| gc.is_nursery_object(addr)) { + return r; + } + majit_gc::gc_sync::is_initialized() + && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_nursery_object(addr)) } /// Returns true when the active GC was present and roots were @@ -14622,14 +14669,7 @@ impl Drop for CraneliftBackend { // active allocator so a subsequent backend is free to install // its own; matching dynasm's // `runner.rs DYNASM_ACTIVE_GC` reset on backend teardown. - // Drop the boxed allocator first so reentrant - // `gc_owns_object_via_active_runtime` queries from its drop - // body still resolve old-heap addresses through the raw - // mirror, then clear the raw mirror. - let _ = CRANELIFT_ACTIVE_GC.try_with(|cell| { - *cell.borrow_mut() = None; - }); - let _ = CRANELIFT_ACTIVE_GC_RAW.try_with(|raw_cell| raw_cell.set(None)); + gc_box::clear_on_teardown(); let _ = CRANELIFT_JITFRAME_TYPE_ID.try_with(|c| c.set(None)); } } diff --git a/majit/majit-backend-dynasm/Cargo.toml b/majit/majit-backend-dynasm/Cargo.toml index 59df4b540f8..157bd0d8e29 100644 --- a/majit/majit-backend-dynasm/Cargo.toml +++ b/majit/majit-backend-dynasm/Cargo.toml @@ -21,5 +21,6 @@ indexmap = { workspace = true } rustc-hash = { workspace = true } [dev-dependencies] +majit-gc = { workspace = true, features = ["gc_box"] } majit-ir = { workspace = true, features = ["test-support"] } majit-trace = { workspace = true } diff --git a/majit/majit-backend-dynasm/src/runner.rs b/majit/majit-backend-dynasm/src/runner.rs index 232d42a3af2..dc01367ce8b 100644 --- a/majit/majit-backend-dynasm/src/runner.rs +++ b/majit/majit-backend-dynasm/src/runner.rs @@ -98,21 +98,141 @@ pub(crate) fn lookup_call_assembler_callee_locs( }) } -thread_local! { - /// llmodel.py self.gc_ll_descr — owned by the active dynasm - /// backend on this thread. Stored as a thread-local so the - /// backend-agnostic `majit_gc::ActiveGcGuardHooks` shims can - /// reach the live allocator without taking a dynasm dependency. - pub static DYNASM_ACTIVE_GC: RefCell>> = - const { RefCell::new(None) }; - static DYNASM_ACTIVE_GC_RAW: std::cell::Cell> = - const { std::cell::Cell::new(None) }; +/// The per-thread GC box, and the accessors every trampoline reaches it through. +/// +/// `gc.py:30` `GcLLDescription.__init__` holds `self.gcdescr` as a plain field +/// on the backend descriptor — there is no per-thread allocator upstream — so +/// this cell is scaffolding, not a ported structure. Only `install_gc_box` +/// fills it and only tests reach that; the production build goes through +/// `install_gc_standalone` and allocates from the `gc_sync` singleton. +/// +/// Every accessor opens with `majit_gc::gc_box_installed()`, which without +/// `majit-gc/gc_box` is a constant `false` — so in a production build each one +/// folds to `None`, the thread-local becomes unreachable, and the trampolines +/// call `gc_sync` directly. The gate lives in `majit-gc` because a Cargo +/// feature is per-crate: this crate cannot `#[cfg]` on a feature of its +/// dependency, so the box is eliminated by the optimizer rather than by +/// conditional compilation. +mod gc_box { + use std::cell::RefCell; + + thread_local! { + /// llmodel.py self.gc_ll_descr — owned by the active dynasm + /// backend on this thread. Stored as a thread-local so the + /// backend-agnostic `majit_gc::ActiveGcGuardHooks` shims can + /// reach the live allocator without taking a dynasm dependency. + pub static DYNASM_ACTIVE_GC: RefCell>> = + const { RefCell::new(None) }; + /// Read-only mirror of the box address, for the queries that can fire + /// while an in-progress allocation already holds the mutable borrow. + static DYNASM_ACTIVE_GC_RAW: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + } + + /// Apply `f` to this thread's GC box. `None` means there is no box, and + /// the caller runs its `gc_sync` path instead. + pub(super) fn with_ref(f: impl FnOnce(&dyn majit_gc::GcAllocator) -> R) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + DYNASM_ACTIVE_GC.with(|cell| cell.borrow().as_deref().map(f)) + } + + /// `&mut` counterpart of [`with_ref`], for allocation and write barriers. + pub(super) fn with_mut(f: impl FnOnce(&mut dyn majit_gc::GcAllocator) -> R) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + DYNASM_ACTIVE_GC.with(|cell| { + let mut guard = cell.borrow_mut(); + let raw: *mut dyn majit_gc::GcAllocator = guard.as_deref_mut()?; + // SAFETY: `guard` holds the `RefCell` borrow for the whole `f` + // call, and these callers are non-reentrant top-level mutator + // trampolines, so the reborrow is exclusive and outlives `f`. The + // raw round-trip is what lets the boxed `dyn + 'static` allocator + // satisfy the `FnOnce(&mut dyn GcAllocator)` HRTB bound (same + // shape `gc_op` gets from its `&'static mut` singleton). + Some(f(unsafe { &mut *raw })) + }) + } + + /// Read-only access that tolerates being reached from inside a collection. + /// + /// Structural adaptation: RPython's GC descriptor is a normal object + /// reference, so `gc_current_object_address` can query ownership while a + /// collection is already walking extra roots. Here the box sits behind a + /// `RefCell` whose mutable borrow an in-progress allocation may hold, so + /// that case reads the same allocator through the raw mirror rather than + /// panicking across the extern slowpath. + pub(super) fn with_reentrant_ref( + f: impl FnOnce(&dyn majit_gc::GcAllocator) -> R, + ) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + DYNASM_ACTIVE_GC.with(|cell| match cell.try_borrow() { + Ok(guard) => guard.as_deref().map(f), + // SAFETY: the mirror is published and cleared under the same + // borrow as the box itself, so a non-null value points at the + // live allocator, and this query only reads it. + Err(_) => DYNASM_ACTIVE_GC_RAW.with(|raw| raw.get().map(|p| f(unsafe { &*p }))), + }) + } + + /// `&mut` access for a top-level, never-reentrant op that has a defined + /// answer when the box is busy: a borrow already held by an in-progress + /// allocation yields `busy` rather than falling through to `gc_sync`. + pub(super) fn with_mut_or_busy( + busy: R, + f: impl FnOnce(&mut dyn majit_gc::GcAllocator) -> R, + ) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + DYNASM_ACTIVE_GC.with(|cell| { + let mut guard = match cell.try_borrow_mut() { + Ok(guard) => guard, + Err(_) => return Some(busy), + }; + let raw: *mut dyn majit_gc::GcAllocator = guard.as_deref_mut()?; + // SAFETY: as in [`with_mut`]. + Some(f(unsafe { &mut *raw })) + }) + } + + /// Whether this thread holds a box at all. + pub(super) fn present() -> bool { + majit_gc::gc_box_installed() && DYNASM_ACTIVE_GC.with(|cell| cell.borrow().is_some()) + } + + /// Store `gc` as this thread's box, publishing the raw mirror with it. + pub(super) fn store(gc: Box) { + DYNASM_ACTIVE_GC.with(|cell| { + let mut guard = cell.borrow_mut(); + *guard = Some(gc); + let raw = guard + .as_deref_mut() + .map(|g| g as *mut dyn majit_gc::GcAllocator); + DYNASM_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(raw)); + }); + } + + /// Drop this thread's box and clear the raw mirror. + /// + /// The box goes first so reentrant ownership queries issued from its drop + /// body still resolve old-heap addresses through the mirror. + pub(super) fn clear() { + DYNASM_ACTIVE_GC.with(|cell| { + *cell.borrow_mut() = None; + }); + DYNASM_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(None)); + } } /// Read-only GC query for the guard hooks and codegen helpers. /// -/// - **Test box present** (`DYNASM_ACTIVE_GC` is `Some`): a test owns the -/// GC directly — apply `f` to the boxed allocator. +/// - **Test box present**: a test owns the GC directly — apply `f` to the +/// boxed allocator. /// - **No box, `gc_sync` initialized** (production): route to the /// process-global singleton via `gc_query_reentrant`. These hooks can /// fire during a collection's guard evaluation / extra-root walk, so @@ -122,9 +242,7 @@ thread_local! { /// returns `None` so callers keep their existing `.unwrap_or(default)` /// / `.flatten()` behaviour. pub(crate) fn with_dynasm_active_gc(f: impl Fn(&dyn majit_gc::GcAllocator) -> R) -> Option { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|cell| cell.borrow().as_deref().map(&f)) - { + if let Some(r) = gc_box::with_ref(&f) { return Some(r); } if majit_gc::gc_sync::is_initialized() { @@ -137,8 +255,8 @@ pub(crate) fn with_dynasm_active_gc(f: impl Fn(&dyn majit_gc::GcAllocator) -> /// GC write-barrier descriptor for machine-code generation. /// -/// Compilation may run outside the mutator thread that owns -/// `DYNASM_ACTIVE_GC`. PyPy keeps this descriptor on `cpu.gc_ll_descr`; use +/// Compilation may run outside the mutator thread that owns the GC box. +/// PyPy keeps this descriptor on `cpu.gc_ll_descr`; use /// the current MiniMark layout in that case instead of silently omitting every /// barrier. If an active collector explicitly reports no descriptor, preserve /// that choice. @@ -160,19 +278,8 @@ pub(crate) fn dynasm_write_barrier_descr() -> Option( f: impl FnOnce(&mut dyn majit_gc::GcAllocator) -> R, ) -> Option { - if DYNASM_ACTIVE_GC.with(|cell| cell.borrow().is_some()) { - return DYNASM_ACTIVE_GC.with(|cell| { - let mut guard = cell.borrow_mut(); - let raw: *mut dyn majit_gc::GcAllocator = guard.as_deref_mut()?; - // SAFETY: `guard` holds the `RefCell` borrow for the whole - // `f` call, and these callers are non-reentrant top-level - // mutator trampolines, so the reborrow is exclusive and - // outlives `f`. The raw round-trip is what lets the boxed - // `dyn + 'static` allocator satisfy the `FnOnce(&mut dyn - // GcAllocator)` HRTB bound (same shape `gc_op` gets from its - // `&'static mut` singleton). - Some(f(unsafe { &mut *raw })) - }); + if gc_box::present() { + return gc_box::with_mut(f); } if majit_gc::gc_sync::is_initialized() { return Some(majit_gc::gc_sync::gc_op(|gc| f(gc))); @@ -245,41 +352,25 @@ fn install_gc_box(gc: Box) { majit_gc::disarm_published_nursery(); majit_gc::note_gc_box_installed(); let supports_guard_gc_type = gc.supports_guard_gc_type(); - DYNASM_ACTIVE_GC.with(|cell| { - let mut guard = cell.borrow_mut(); - *guard = Some(gc); - let raw = guard - .as_deref_mut() - .map(|g| g as *mut dyn majit_gc::GcAllocator); - DYNASM_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(raw)); - }); + gc_box::store(gc); register_active_hooks(supports_guard_gc_type); } /// Production path: register all `set_active_*` hooks WITHOUT storing a -/// box. `DYNASM_ACTIVE_GC` stays `None`, so every trampoline routes to -/// the process-global `gc_sync` singleton (the per-thread GC box is the -/// free-threading gap R4 removes). +/// box, so every trampoline routes to the process-global `gc_sync` singleton +/// (the per-thread GC box is the free-threading gap R4 removes). pub fn install_gc_standalone() { majit_gc::gc_sync::gc_op(|gc| gc.freeze_types()); let supports_guard_gc_type = majit_gc::gc_sync::gc_query(|gc| gc.supports_guard_gc_type()); register_active_hooks(supports_guard_gc_type); } -/// Clear both `DYNASM_ACTIVE_GC` and `DYNASM_ACTIVE_GC_RAW`. Callers -/// that want to drop the active dynasm GC must go through this helper -/// rather than mutating `DYNASM_ACTIVE_GC` directly, otherwise the raw -/// mirror used by `dynasm_gc_owns_object`'s reentrant fallback would be -/// left pointing at freed memory. +/// Drop the active dynasm GC box. Callers must go through this helper rather +/// than reaching the thread-local directly, otherwise the raw mirror used by +/// `dynasm_gc_owns_object`'s reentrant fallback would be left pointing at +/// freed memory. pub fn clear_gc_allocator() { - // Drop the boxed allocator first so reentrant - // `dynasm_gc_owns_object` queries from its drop body still resolve - // old-heap addresses through the raw mirror, then clear the raw - // mirror. - DYNASM_ACTIVE_GC.with(|cell| { - *cell.borrow_mut() = None; - }); - DYNASM_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(None)); + gc_box::clear(); } /// TYPE_INFO / CLASSTYPE constants read by the dynasm assemblers for @@ -378,13 +469,7 @@ pub(crate) fn new_via_gc_enabled() -> bool { /// `gc_alloc_nursery_shim`), including the process-global singleton; falls back /// to `malloc` when no GC is installed. pub(crate) extern "C" fn dynasm_new_alloc(size: usize) -> *mut u8 { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|cell| { - cell.borrow_mut() - .as_deref_mut() - .map(|gc| gc.alloc_nursery(size)) - }) - { + if let Some(r) = gc_box::with_mut(|gc| gc.alloc_nursery(size)) { return r.0 as *mut u8; } if majit_gc::gc_sync::is_initialized() { @@ -401,13 +486,7 @@ pub(crate) extern "C" fn dynasm_new_alloc(size: usize) -> *mut u8 { /// rather than the host heap, where its collector could not see it. Returns /// null when no GC is bound, leaving the caller on its own path. fn dynasm_alloc_nursery_headerless_no_collect(size: usize) -> GcRef { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.alloc_nursery_headerless_no_collect(size)) - }) - { + if let Some(r) = gc_box::with_mut(|g| g.alloc_nursery_headerless_no_collect(size)) { return r; } if majit_gc::gc_sync::is_initialized() { @@ -431,13 +510,7 @@ fn dynasm_alloc_nursery_typed(type_id: u32, size: usize) -> GcRef { // collections that fire between here and the caller's store into a // tracked slot — and returns NULL when rawmalloc fails so the host helper // can raise `MemoryError`. - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.try_alloc_nursery_no_collect_typed(type_id, size)) - }) - { + if let Some(r) = gc_box::with_mut(|g| g.try_alloc_nursery_no_collect_typed(type_id, size)) { return r; } majit_gc::standalone_alloc_nursery_typed(type_id, size) @@ -453,17 +526,9 @@ unsafe fn dynasm_alloc_nursery_typed_with_placement( size: usize, needs_write_barrier: *mut bool, ) -> GcRef { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut().as_deref_mut().map(|g| unsafe { - g.try_alloc_nursery_no_collect_typed_with_placement( - type_id, - size, - needs_write_barrier, - ) - }) - }) - { + if let Some(r) = gc_box::with_mut(|g| unsafe { + g.try_alloc_nursery_no_collect_typed_with_placement(type_id, size, needs_write_barrier) + }) { return r; } majit_gc::gc_sync::gc_op(|g| unsafe { @@ -479,13 +544,7 @@ unsafe fn dynasm_alloc_nursery_typed_with_placement( /// allocation — so the embedded minor cycle is safe and dead bigints are /// reclaimed instead of accumulating in old-gen. fn dynasm_alloc_nursery_collecting_typed(type_id: u32, size: usize) -> GcRef { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.alloc_nursery_typed(type_id, size)) - }) - { + if let Some(r) = gc_box::with_mut(|g| g.alloc_nursery_typed(type_id, size)) { return r; } majit_gc::gc_sync::gc_op(|g| g.alloc_nursery_typed(type_id, size)) @@ -505,13 +564,9 @@ unsafe fn dynasm_alloc_nursery_collecting_typed_rooted( root: *mut GcRef, needs_write_barrier: *mut bool, ) -> GcRef { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut().as_deref_mut().map(|g| unsafe { - g.alloc_nursery_collecting_typed_rooted(type_id, size, root, needs_write_barrier) - }) - }) - { + if let Some(r) = gc_box::with_mut(|g| unsafe { + g.alloc_nursery_collecting_typed_rooted(type_id, size, root, needs_write_barrier) + }) { return r; } unsafe { @@ -531,13 +586,7 @@ unsafe fn dynasm_alloc_nursery_collecting_typed_rooted( /// (non-moving), so the returned pointer is stable across minor and /// major collections. fn dynasm_alloc_oldgen_typed(type_id: u32, size: usize) -> GcRef { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.alloc_oldgen_typed(type_id, size)) - }) - { + if let Some(r) = gc_box::with_mut(|g| g.alloc_oldgen_typed(type_id, size)) { return r; } majit_gc::gc_sync::gc_op(|g| g.alloc_oldgen_typed(type_id, size)) @@ -594,10 +643,7 @@ fn bh_alloc_struct(sizedescr: &majit_translate::jitcode::BhDescr) -> *mut libc:: /// Python value stack, or on the shadow stack — Rust-stack PyObjectRef /// in nursery would dangle after the embedded minor cycle. fn dynasm_collect_full() { - if DYNASM_ACTIVE_GC - .with(|c| c.borrow_mut().as_deref_mut().map(|g| g.collect_full())) - .is_some() - { + if gc_box::with_mut(|g| g.collect_full()).is_some() { return; } majit_gc::gc_sync::gc_op(|g| g.collect_full()); @@ -611,14 +657,7 @@ fn debug_validate_oldgen_freeblocks(site: std::fmt::Arguments<'_>) { return; } let site = site.to_string(); - if DYNASM_ACTIVE_GC - .with(|c| { - c.borrow() - .as_deref() - .map(|g| g.debug_validate_oldgen_freeblocks(&site)) - }) - .is_some() - { + if gc_box::with_ref(|g| g.debug_validate_oldgen_freeblocks(&site)).is_some() { return; } majit_gc::gc_sync::gc_op(|g| g.debug_validate_oldgen_freeblocks(&site)); @@ -642,14 +681,7 @@ pub extern "C" fn dynasm_debug_validate_oldgen_freeblocks(site: u64, frame: usiz fn dynasm_get_objects(generation: i8, visitor: majit_gc::GetObjectsVisitorFn) { let mut visit = visitor; - if DYNASM_ACTIVE_GC - .with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.get_objects(generation, &mut visit)) - }) - .is_some() - { + if gc_box::with_mut(|g| g.get_objects(generation, &mut visit)).is_some() { return; } majit_gc::gc_sync::gc_op(|g| g.get_objects(generation, &mut visit)); @@ -657,14 +689,7 @@ fn dynasm_get_objects(generation: i8, visitor: majit_gc::GetObjectsVisitorFn) { fn dynasm_get_referents(obj: majit_ir::GcRef, visitor: majit_gc::GetObjectsVisitorFn) { let mut visit = visitor; - if DYNASM_ACTIVE_GC - .with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.get_referents(obj, &mut visit)) - }) - .is_some() - { + if gc_box::with_mut(|g| g.get_referents(obj, &mut visit)).is_some() { return; } // See `MajitGc::get_referents`: the fallback can park, so the argument is @@ -673,9 +698,7 @@ fn dynasm_get_referents(obj: majit_ir::GcRef, visitor: majit_gc::GetObjectsVisit } fn dynasm_is_tracked(obj: majit_ir::GcRef) -> bool { - if let Some(tracked) = - DYNASM_ACTIVE_GC.with(|c| c.borrow_mut().as_deref_mut().map(|g| g.is_tracked(obj))) - { + if let Some(tracked) = gc_box::with_mut(|g| g.is_tracked(obj)) { return tracked; } majit_gc::gc_sync::gc_op_with_root(obj, |g, obj| g.is_tracked(obj)) @@ -686,14 +709,7 @@ fn dynasm_is_tracked(obj: majit_ir::GcRef) -> bool { /// an active JIT (nursery non-empty) — unlike [`dynasm_collect_full`], whose /// embedded minor would relocate a Rust-stack nursery PyObjectRef. fn dynasm_collect_oldgen_nonmoving() { - if DYNASM_ACTIVE_GC - .with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.collect_oldgen_nonmoving()) - }) - .is_some() - { + if gc_box::with_mut(|g| g.collect_oldgen_nonmoving()).is_some() { return; } majit_gc::gc_sync::gc_op(|g| g.collect_oldgen_nonmoving()); @@ -709,9 +725,7 @@ fn dynasm_finalizer_next_dead(fq_index: usize) -> Option { /// Report `(oldgen_total, nursery_used)` for the interpreter GC safepoint. fn dynasm_heap_stats() -> (usize, usize) { - if let Some(r) = - DYNASM_ACTIVE_GC.with(|c| c.borrow_mut().as_deref_mut().map(|g| g.heap_byte_stats())) - { + if let Some(r) = gc_box::with_mut(|g| g.heap_byte_stats()) { return r; } majit_gc::gc_sync::gc_op(|g| g.heap_byte_stats()) @@ -720,13 +734,7 @@ fn dynasm_heap_stats() -> (usize, usize) { /// Report whether the GC wants a major collection, for the interpreter GC /// safepoint (incminimark.py:1288-1290 `threshold_reached`). fn dynasm_major_threshold_reached() -> bool { - if majit_gc::gc_box_installed() - && let Some(r) = DYNASM_ACTIVE_GC.with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.major_threshold_reached()) - }) - { + if let Some(r) = gc_box::with_mut(|g| g.major_threshold_reached()) { return r; } majit_gc::gc_sync::gc_op(|g| g.major_threshold_reached()) @@ -739,14 +747,7 @@ fn dynasm_major_threshold_reached() -> bool { /// Caller must keep `slot` valid until [`dynasm_gc_remove_root`] is /// called with the same pointer. unsafe fn dynasm_gc_add_root(slot: *mut GcRef) { - if DYNASM_ACTIVE_GC - .with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| unsafe { g.add_root(slot) }) - }) - .is_some() - { + if gc_box::with_mut(|g| unsafe { g.add_root(slot) }).is_some() { return; } unsafe { majit_gc::gc_sync::gc_op_add_root(slot) }; @@ -754,10 +755,7 @@ unsafe fn dynasm_gc_add_root(slot: *mut GcRef) { /// Companion to [`dynasm_gc_add_root`]. fn dynasm_gc_remove_root(slot: *mut GcRef) { - if DYNASM_ACTIVE_GC - .with(|c| c.borrow_mut().as_deref_mut().map(|g| g.remove_root(slot))) - .is_some() - { + if gc_box::with_mut(|g| g.remove_root(slot)).is_some() { return; } majit_gc::gc_sync::gc_op(|g| g.remove_root(slot)); @@ -766,43 +764,24 @@ fn dynasm_gc_remove_root(slot: *mut GcRef) { /// Host-side write-barrier trampoline for GC-managed objects updated /// outside compiled code. fn dynasm_gc_write_barrier(obj: GcRef) { - if DYNASM_ACTIVE_GC - .with(|c| c.borrow_mut().as_deref_mut().map(|g| g.write_barrier(obj))) - .is_some() - { + if gc_box::with_mut(|g| g.write_barrier(obj)).is_some() { return; } majit_gc::gc_sync::gc_op_with_root(obj, |g, obj| g.write_barrier(obj)); } fn dynasm_gc_write_barrier_managed(obj: GcRef) { - if DYNASM_ACTIVE_GC - .with(|c| { - c.borrow_mut() - .as_deref_mut() - .map(|g| g.write_barrier_managed(obj)) - }) - .is_some() - { + if gc_box::with_mut(|g| g.write_barrier_managed(obj)).is_some() { return; } majit_gc::gc_sync::gc_op_with_root(obj, |g, obj| g.write_barrier_managed(obj)); } fn dynasm_id_or_identityhash(addr: usize) -> usize { - // Keep the defensive `try_borrow_mut`: a borrow already held by an - // in-progress alloc means the test box is busy, so fall back to the - // raw `addr` (this is a top-level op, never reentrant). - let via_box = majit_gc::gc_box_installed().then(|| { - DYNASM_ACTIVE_GC.with(|cell| { - let mut guard = match cell.try_borrow_mut() { - Ok(guard) => guard, - Err(_) => return Some(addr), - }; - guard.as_deref_mut().map(|gc| gc.id_or_identityhash(addr)) - }) - }); - if let Some(Some(r)) = via_box { + // A box whose borrow is already held by an in-progress alloc answers with + // the raw `addr`, not with the singleton's id: this is a top-level op, so + // the busy borrow means the box is mid-allocation, not that it is absent. + if let Some(r) = gc_box::with_mut_or_busy(addr, |gc| gc.id_or_identityhash(addr)) { return r; } majit_gc::gc_sync::gc_op(|g| g.id_or_identityhash(addr)) @@ -815,65 +794,22 @@ fn dynasm_id_or_identityhash(addr: usize) -> usize { /// `false` when no GC is installed (caller falls through to /// `std::alloc::dealloc`). fn dynasm_gc_owns_object(addr: usize) -> bool { - // Structural adaptation: RPython's GC descriptor is a normal object - // reference and `gc_current_object_address` can query ownership while - // a collection is already in progress. Pyre stores the active dynasm - // GC behind a Rust `RefCell` (test box) or in the process-global - // `gc_sync` singleton (production). This read-only ownership query can - // fire reentrantly from an extra-root walker mid-collection, so: - // - `Ok(Some)`: a test box is present and free-borrowable → use it. - // - `Ok(None)`: no box (production) → route to `gc_sync` reentrantly. - // - `Err`: the test box's mutable borrow is held by an in-progress - // alloc → reach it through the raw mirror (read-only), or fall - // back to `gc_sync` if there is no box at all. - if !majit_gc::gc_box_installed() { - return majit_gc::gc_sync::is_initialized() - && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)); - } - match DYNASM_ACTIVE_GC.with(|c| { - c.try_borrow() - .map(|g| g.as_deref().map(|gc| gc.is_managed_heap_object(addr))) - }) { - Ok(Some(r)) => r, - Ok(None) => { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) - } else { - false - } - } - Err(_) => DYNASM_ACTIVE_GC_RAW.with(|raw| match raw.get() { - Some(p) => unsafe { (&*p).is_managed_heap_object(addr) }, - None => { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) - } else { - false - } - } - }), + // This query can fire reentrantly from an extra-root walker mid-collection, + // so both arms are read-only: `with_reentrant_ref` for the box, and the + // reentrant singleton read for everything else. + if let Some(r) = gc_box::with_reentrant_ref(|gc| gc.is_managed_heap_object(addr)) { + return r; } + majit_gc::gc_sync::is_initialized() + && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) } fn dynasm_gc_is_nursery_object(addr: usize) -> bool { - let via_box = majit_gc::gc_box_installed() - .then(|| { - DYNASM_ACTIVE_GC.with(|cell| match cell.try_borrow() { - Ok(guard) => guard.as_deref().map(|gc| gc.is_nursery_object(addr)), - Err(_) => DYNASM_ACTIVE_GC_RAW.with(|raw| { - raw.get() - .map(|ptr| unsafe { (&*ptr).is_nursery_object(addr) }) - }), - }) - }) - .flatten(); - via_box.unwrap_or_else(|| { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_nursery_object(addr)) - } else { - false - } - }) + if let Some(r) = gc_box::with_reentrant_ref(|gc| gc.is_nursery_object(addr)) { + return r; + } + majit_gc::gc_sync::is_initialized() + && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_nursery_object(addr)) } /// `gc.py:51` malloc-helper OOM signaling. @@ -4151,7 +4087,7 @@ mod tests { const TEST_HELPER_MARKER: i64 = 0x5a5a5a5a_i64; extern "C" fn alloc_marked_ref() -> i64 { - DYNASM_ACTIVE_GC.with(|cell| { + crate::runner::gc_box::DYNASM_ACTIVE_GC.with(|cell| { let mut guard = cell.borrow_mut(); let gc = guard .as_mut() @@ -4167,7 +4103,7 @@ mod tests { } extern "C" fn alloc_marked_ref_collecting() -> i64 { - DYNASM_ACTIVE_GC.with(|cell| { + crate::runner::gc_box::DYNASM_ACTIVE_GC.with(|cell| { let mut guard = cell.borrow_mut(); let gc = guard .as_mut() diff --git a/majit/majit-backend-wasm/Cargo.toml b/majit/majit-backend-wasm/Cargo.toml index a721b4d167c..011e2d8c34d 100644 --- a/majit/majit-backend-wasm/Cargo.toml +++ b/majit/majit-backend-wasm/Cargo.toml @@ -31,6 +31,7 @@ wasm-encoder = { workspace = true } wasm-bindgen = { workspace = true, optional = true } [dev-dependencies] +majit-gc = { workspace = true, features = ["gc_box"] } majit-ir = { workspace = true, features = ["test-support"] } wasmparser = { workspace = true } smallvec = { workspace = true } diff --git a/majit/majit-backend-wasm/src/lib.rs b/majit/majit-backend-wasm/src/lib.rs index 5d6915d1413..6989debc638 100644 --- a/majit/majit-backend-wasm/src/lib.rs +++ b/majit/majit-backend-wasm/src/lib.rs @@ -288,40 +288,96 @@ pub fn jit_call_area_addr() -> usize { &JIT_CALL_AREA as *const _ as usize } -thread_local! { - /// llmodel.py self.gc_ll_descr — owned by the active wasm - /// backend on this thread. Stored as a thread-local so the - /// backend-agnostic `majit_gc::ActiveGcGuardHooks` shims can - /// reach the live allocator without taking a wasm dependency. - /// Mirrors `cranelift::compiler::CRANELIFT_ACTIVE_GC` and - /// `dynasm::runner::DYNASM_ACTIVE_GC` — RPython's - /// `cpu.gc_ll_descr` parity, single-slot per thread. - static WASM_ACTIVE_GC: RefCell>> = const { RefCell::new(None) }; - /// Raw mirror of the boxed allocator, read by `wasm_gc_owns_object`'s - /// reentrant fallback: the interpreter-safepoint major holds the - /// `WASM_ACTIVE_GC` mutable borrow while extra-root walkers ask whether a - /// slot is GC-managed, so that query routes through the raw pointer instead - /// of a second borrow. Mirrors `dynasm::runner::DYNASM_ACTIVE_GC_RAW`. - static WASM_ACTIVE_GC_RAW: std::cell::Cell> = - const { std::cell::Cell::new(None) }; +/// The per-thread GC box, and the accessors every trampoline reaches it through. +/// +/// `gc.py:30` `GcLLDescription.__init__` holds `self.gcdescr` as a plain field +/// on the backend descriptor — there is no per-thread allocator upstream — so +/// this cell is scaffolding, not a ported structure. Only `install_gc_box` +/// fills it and only tests reach that; the production build goes through +/// `install_gc_standalone` and allocates from the `gc_sync` singleton. +/// +/// Every accessor opens with `majit_gc::gc_box_installed()`, which without +/// `majit-gc/gc_box` is a constant `false` — so in a production build each one +/// folds to `None`, the thread-local becomes unreachable, and the trampolines +/// call `gc_sync` directly. The gate lives in `majit-gc` because a Cargo +/// feature is per-crate: this crate cannot `#[cfg]` on a feature of its +/// dependency, so the box is eliminated by the optimizer rather than by +/// conditional compilation. Mirrors `majit-backend-dynasm/src/runner.rs`'s +/// `gc_box`. +mod gc_box { + use super::{GcAllocator, RefCell}; + + thread_local! { + /// llmodel.py self.gc_ll_descr — owned by the active wasm backend on + /// this thread. Stored as a thread-local so the backend-agnostic + /// `majit_gc::ActiveGcGuardHooks` shims can reach the live allocator + /// without taking a wasm dependency. RPython's `cpu.gc_ll_descr` + /// parity, single-slot per thread. + static WASM_ACTIVE_GC: RefCell>> = + const { RefCell::new(None) }; + /// Read-only mirror of the box address: the interpreter-safepoint major + /// holds the mutable borrow while extra-root walkers ask whether a slot + /// is GC-managed, so that query routes through the raw pointer instead + /// of taking a second borrow. + static WASM_ACTIVE_GC_RAW: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + } + + /// `&mut` access to this thread's GC box, for allocation, write barriers + /// and collection. `None` means there is no box, and the caller runs its + /// `gc_sync` path instead. + pub(super) fn with_mut(f: impl FnOnce(&mut dyn GcAllocator) -> R) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + WASM_ACTIVE_GC.with(|cell| { + let mut guard = cell.borrow_mut(); + let raw: *mut dyn GcAllocator = guard.as_deref_mut()?; + // SAFETY: `guard` holds the borrow for the whole `f` call and + // these are non-reentrant top-level trampolines, so the reborrow + // is exclusive and outlives `f`. + Some(f(unsafe { &mut *raw })) + }) + } + + /// Read-only access that tolerates being reached from inside a collection: + /// when an in-progress mutation already holds the mutable borrow, read the + /// same allocator through the raw mirror rather than taking a second one. + pub(super) fn with_reentrant_ref(f: impl FnOnce(&dyn GcAllocator) -> R) -> Option { + if !majit_gc::gc_box_installed() { + return None; + } + WASM_ACTIVE_GC.with(|cell| match cell.try_borrow() { + Ok(guard) => guard.as_deref().map(f), + // SAFETY: the mirror is published and cleared under the same + // borrow as the box itself, so a non-null value points at the + // live allocator, and this query only reads it. + Err(_) => WASM_ACTIVE_GC_RAW.with(|raw| raw.get().map(|p| f(unsafe { &*p }))), + }) + } + + /// Whether this thread holds a box at all. + pub(super) fn present() -> bool { + majit_gc::gc_box_installed() && WASM_ACTIVE_GC.with(|cell| cell.borrow().is_some()) + } + + /// Store `gc` as this thread's box, publishing the raw mirror with it. + pub(super) fn store(gc: Box) { + WASM_ACTIVE_GC.with(|cell| { + let mut guard = cell.borrow_mut(); + *guard = Some(gc); + let raw = guard.as_deref_mut().map(|gc| gc as *mut dyn GcAllocator); + WASM_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(raw)); + }); + } } +/// Read-only GC query for the guard hooks and codegen helpers. The box arm is +/// reentrancy-tolerant because these can fire during a collection's extra-root +/// walk, which is also why the singleton arm is the reentrant read. fn with_wasm_active_gc(f: impl Fn(&dyn GcAllocator) -> R) -> Option { - if !majit_gc::gc_box_installed() { - return majit_gc::gc_sync::is_initialized() - .then(|| majit_gc::gc_sync::gc_query_reentrant(|gc| f(gc))); - } - match WASM_ACTIVE_GC.with(|cell| cell.try_borrow().map(|g| g.as_deref().map(|gc| f(gc)))) { - Ok(Some(r)) => return Some(r), - Ok(None) => {} - Err(_) => { - if let Some(ptr) = WASM_ACTIVE_GC_RAW.with(|raw| raw.get()) { - // SAFETY: the raw mirror points at the same live test box whose - // mutable borrow is held by the in-progress mutation. This is a - // read-only reentrant query, so it does not create a second &mut. - return Some(f(unsafe { &*ptr })); - } - } + if let Some(r) = gc_box::with_reentrant_ref(&f) { + return Some(r); } if majit_gc::gc_sync::is_initialized() { return Some(majit_gc::gc_sync::gc_query_reentrant(|gc| f(gc))); @@ -335,15 +391,8 @@ fn with_wasm_active_gc(f: impl Fn(&dyn GcAllocator) -> R) -> Option { /// `None` so callers keep their non-GC fallback. Top-level mutator/ /// blackhole trampolines, never inside a collection, so `gc_op` is correct. fn with_wasm_active_gc_mut(f: impl FnOnce(&mut dyn GcAllocator) -> R) -> Option { - if majit_gc::gc_box_installed() && WASM_ACTIVE_GC.with(|cell| cell.borrow().is_some()) { - return WASM_ACTIVE_GC.with(|cell| { - let mut guard = cell.borrow_mut(); - let raw: *mut dyn GcAllocator = guard.as_deref_mut()?; - // SAFETY: `guard` holds the borrow for the whole `f` call and - // these are non-reentrant top-level trampolines, so the - // reborrow is exclusive and outlives `f`. - Some(f(unsafe { &mut *raw })) - }); + if gc_box::present() { + return gc_box::with_mut(f); } if majit_gc::gc_sync::is_initialized() { return Some(majit_gc::gc_sync::gc_op(|gc| f(gc))); @@ -409,12 +458,7 @@ fn install_gc_box(gc: Box) { majit_gc::disarm_published_nursery(); majit_gc::note_gc_box_installed(); let supports_guard_gc_type = gc.supports_guard_gc_type(); - WASM_ACTIVE_GC.with(|cell| { - let mut guard = cell.borrow_mut(); - *guard = Some(gc); - let raw = guard.as_deref_mut().map(|gc| gc as *mut dyn GcAllocator); - WASM_ACTIVE_GC_RAW.with(|raw_cell| raw_cell.set(raw)); - }); + gc_box::store(gc); register_active_hooks(supports_guard_gc_type); } @@ -1015,43 +1059,15 @@ fn wasm_active_gc_write_barrier(obj: GcRef) { /// Host-side `is_managed_heap_object` trampoline. /// -/// This read-only ownership query can fire reentrantly from an extra-root -/// walker mid-collection (the interpreter-safepoint major holds the -/// `WASM_ACTIVE_GC` mutable borrow while asking whether a slot is -/// GC-managed), so: -/// - `Ok(Some)`: a test box is present and free-borrowable → use it. -/// - `Ok(None)`: no box (production) → route to `gc_sync` reentrantly. -/// - `Err`: the test box's mutable borrow is held by an in-progress -/// mutation → reach it through the raw mirror (read-only), or fall -/// back to `gc_sync` if there is no box at all. +/// This query can fire reentrantly from an extra-root walker mid-collection +/// (the interpreter-safepoint major holds the box's mutable borrow while +/// asking whether a slot is GC-managed), so both arms are read-only. fn wasm_gc_owns_object(addr: usize) -> bool { - if !majit_gc::gc_box_installed() { - return majit_gc::gc_sync::is_initialized() - && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)); - } - match WASM_ACTIVE_GC.with(|cell| { - cell.try_borrow() - .map(|g| g.as_deref().map(|gc| gc.is_managed_heap_object(addr))) - }) { - Ok(Some(r)) => r, - Ok(None) => { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) - } else { - false - } - } - Err(_) => WASM_ACTIVE_GC_RAW.with(|raw| match raw.get() { - Some(ptr) => unsafe { (*ptr).is_managed_heap_object(addr) }, - None => { - if majit_gc::gc_sync::is_initialized() { - majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) - } else { - false - } - } - }), + if let Some(r) = gc_box::with_reentrant_ref(|gc| gc.is_managed_heap_object(addr)) { + return r; } + majit_gc::gc_sync::is_initialized() + && majit_gc::gc_sync::gc_query_reentrant(|g| g.is_managed_heap_object(addr)) } pub struct WasmBackend { diff --git a/majit/majit-gc/Cargo.toml b/majit/majit-gc/Cargo.toml index d75a391fc64..17a9471767c 100644 --- a/majit/majit-gc/Cargo.toml +++ b/majit/majit-gc/Cargo.toml @@ -27,6 +27,16 @@ gc_stress = [] # by default so ordinary wasm builds do not abort. bh_probe = [] +# Per-thread GC box. `install_gc_box` stores a caller-supplied `GcAllocator` in +# a backend thread-local and every allocation, write-barrier and heap query +# checks for it first. Only tests install one — production goes through +# `install_gc_standalone` and allocates from the `gc_sync` singleton — so +# without this feature `gc_box_installed()` is a constant `false` and the +# backends' box branches fold away entirely. A crate whose tests install a box +# must carry `majit-gc = { ..., features = ["gc_box"] }` in `dev-dependencies`; +# installing one without it panics rather than silently missing every probe. +gc_box = [] + [dependencies] indexmap = { workspace = true } majit-ir = { workspace = true } diff --git a/majit/majit-gc/src/lib.rs b/majit/majit-gc/src/lib.rs index e82ba18d419..a772a41bc94 100644 --- a/majit/majit-gc/src/lib.rs +++ b/majit/majit-gc/src/lib.rs @@ -1626,20 +1626,48 @@ pub fn supports_guard_gc_type() -> bool { /// `malloc` against the single translation-time `gcdata` /// (`gctransform/framework.py`) — so the box is pyre-side test scaffolding, and /// the queries below let the allocation path it does not serve skip it. +/// +/// Only compiled when a box can exist at all, i.e. under `gc_box`; see +/// [`gc_box_installed`] for what the other build spells instead. +#[cfg(any(test, feature = "gc_box"))] static GC_BOX_INSTALLED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); /// Record that a backend has installed a per-thread GC box. Called from each /// backend's box-installing entry point. +#[cfg(any(test, feature = "gc_box"))] pub fn note_gc_box_installed() { GC_BOX_INSTALLED.store(true, std::sync::atomic::Ordering::Release); } +/// A build without `gc_box` answers every [`gc_box_installed`] query with a +/// constant `false`, so a box installed here would be invisible to all of them +/// and its owner would silently allocate out of the singleton instead. Refuse +/// loudly rather than let that read as a pass. +#[cfg(not(any(test, feature = "gc_box")))] +pub fn note_gc_box_installed() { + panic!( + "a per-thread GC box was installed in a build without the `gc_box` feature; \ + production installs the backend standalone (`install_gc_standalone`), and a \ + test that needs a box must depend on `majit-gc` with `features = [\"gc_box\"]`" + ); +} + /// Whether any thread may own a per-thread GC box. See [`GC_BOX_INSTALLED`]. +#[cfg(any(test, feature = "gc_box"))] #[inline] pub fn gc_box_installed() -> bool { GC_BOX_INSTALLED.load(std::sync::atomic::Ordering::Acquire) } +/// No backend can hold a box in this build, so every `gc_box_installed() && …` +/// probe folds away and the allocation, write-barrier and query trampolines +/// call `gc_sync` directly — which is the whole shape RPython has. +#[cfg(not(any(test, feature = "gc_box")))] +#[inline(always)] +pub fn gc_box_installed() -> bool { + false +} + // ── Host-side nursery allocation hook ─────────────────────────────── // // Separate from `ActiveGcGuardHooks` because allocation is not a diff --git a/pyre/bench/synth/getframe_method_call_residual_body_once.cranelift.jitstats b/pyre/bench/synth/getframe_method_call_residual_body_once.cranelift.jitstats new file mode 100644 index 00000000000..51743b89057 --- /dev/null +++ b/pyre/bench/synth/getframe_method_call_residual_body_once.cranelift.jitstats @@ -0,0 +1,14 @@ +bridges_compiled=0 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=10 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=0 +internal_compile_panics=0 +loops_aborted=10 +loops_compiled=0 diff --git a/pyre/bench/synth/getframe_method_call_residual_body_once.dynasm.jitstats b/pyre/bench/synth/getframe_method_call_residual_body_once.dynasm.jitstats new file mode 100644 index 00000000000..51743b89057 --- /dev/null +++ b/pyre/bench/synth/getframe_method_call_residual_body_once.dynasm.jitstats @@ -0,0 +1,14 @@ +bridges_compiled=0 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=10 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=0 +internal_compile_panics=0 +loops_aborted=10 +loops_compiled=0 diff --git a/pyre/bench/synth/getframe_method_call_residual_body_once.py b/pyre/bench/synth/getframe_method_call_residual_body_once.py new file mode 100644 index 00000000000..b833fdf0f6e --- /dev/null +++ b/pyre/bench/synth/getframe_method_call_residual_body_once.py @@ -0,0 +1,57 @@ +# A method-form `LOAD_ATTR` + `CALL` whose callee forces the calling frame and +# then raises, caught in the loop body. The callee's own body is outside every +# journal, so a walk that cannot commit makes the legacy entry replay run it a +# second time per trace attempt: `Force.hits` climbs past the iteration count +# while `total` and `len(seen)` — both journaled — stay correct. That asymmetry +# is the whole oracle; a bench that only checked the loop result saw nothing. +# +# The shape matters, not the effect. `f.raiser()` lowers to `LOAD_ATTR` (method +# form) followed by `CALL`, and the CALL's lowering re-enters the LOAD_ATTR +# source segment, so the walk's py_pc floor reports the earlier opcode again +# after the later one. That backward step is no CFG successor, so it arms the +# out-of-order region and every following boundary reseeds the operand mirror +# from the virtualizable shadow — which mid-expression holds the NULLs the +# in-flight opcode already popped, and so cannot return the callable. The +# mirror came back one slot short with the callable in the `self_or_null` slot, +# the escape flush declined on the hole, and the portal replayed the frame from +# its entry. +# +# N is sized to reach the tracing thresholds on its own rather than for +# throughput: at 200 the loop never gets hot and the bench passes with the +# defect in place. Measured against a binary without the fix, N=2000 reports +# 2001 hits and N=20000 reports 20005 — the overshoot is the trace-attempt +# count, so only the fixed value (exactly N) is stable enough to pin. No +# `max-pypy-ratio` for the same reason `exception_reused_object_tb_not_doubled` +# has none — this is a shape oracle, not a workload. +# +# Expected output: (20000, 20000, 20000) +import sys + +N = 20000 + + +class Force: + hits = 0 + + def raiser(self): + _ = sys._getframe(1).f_back + Force.hits += 1 + raise ValueError(1) + + +f = Force() + + +def main(): + seen = [] + total = 0 + for i in list(range(N)): + try: + total += f.raiser() + except ValueError: + total += 1 + seen.append(i) + return total, len(seen), Force.hits + + +print(main()) diff --git a/pyre/bench/synth/getframe_method_call_residual_body_once.wasm.jitstats b/pyre/bench/synth/getframe_method_call_residual_body_once.wasm.jitstats new file mode 100644 index 00000000000..51743b89057 --- /dev/null +++ b/pyre/bench/synth/getframe_method_call_residual_body_once.wasm.jitstats @@ -0,0 +1,14 @@ +bridges_compiled=0 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=10 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=0 +internal_compile_panics=0 +loops_aborted=10 +loops_compiled=0 diff --git a/pyre/pyre-interpreter/src/module/posix/interp_posix.rs b/pyre/pyre-interpreter/src/module/posix/interp_posix.rs index 6e45f7cafca..1b2cb064bfc 100644 --- a/pyre/pyre-interpreter/src/module/posix/interp_posix.rs +++ b/pyre/pyre-interpreter/src/module/posix/interp_posix.rs @@ -4336,7 +4336,6 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { // `EBADF`. #[cfg(all(windows, feature = "host_env", not(feature = "sandbox")))] { - use std::os::windows::io::{AsRawHandle, FromRawHandle}; let invalid_handle = || { crate::PyError::os_error_win32_syscall2( windows_sys::Win32::Foundation::ERROR_INVALID_HANDLE as i32, @@ -4345,13 +4344,17 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { ) }; let borrowed = unsafe { rustpython_host_env::crt_fd::Borrowed::borrow_raw(fd) }; - let handle = - rustpython_host_env::crt_fd::as_handle(borrowed).map_err(|_| invalid_handle())?; - let file = unsafe { std::fs::File::from_raw_handle(handle.as_raw_handle()) }; - let meta = file.metadata(); - let _ = std::mem::ManuallyDrop::new(file); // don't close - match meta { - Ok(m) => Ok(make_stat_result(&m, 0)), + // Same `StatStruct` the path forms take, for the reason + // [`win_stat_fields`] states: `std::fs::Metadata` carries no file + // index or volume serial, so answering from it would report a + // different identity for the very file `os.stat(path)` just named. + // It also answers a character device or pipe with the format bits + // `GetFileType` reports rather than a disk file's. + match rustpython_host_env::fileutils::fstat(borrowed) { + Ok(st) => Ok(stat_result_from_fields( + &stat_fields_from_statstruct(&st), + 0, + )), Err(e) => Err(match e.raw_os_error() { Some(winerror) => crate::PyError::os_error_win32_syscall2( winerror, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index d4e8b5b9387..5f96ce96a12 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -404,6 +404,7 @@ pub fn dispatch_via_miframe( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -1249,6 +1250,7 @@ pub(crate) fn drive_bridge_frame_subwalk( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 5518ae5b279..6ec033278db 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -4224,6 +4224,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6466,6 +6467,7 @@ pub(crate) fn run_sub_jitcode_walk( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 97fab478dd3..7f6b43906d4 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -1656,6 +1656,19 @@ pub struct WalkContext<'frame, 'static_a: 'frame, Sym: WalkSym> { /// the virtualizable shadow instead of replaying stack effects. Cleared once /// the walk advances past this ceiling (py-pc order is monotonic again). pub vstack_reorder_ceiling: u32, + /// The `(py_pc, depth, boxes)` the mirror held when `vstack_reorder_ceiling` + /// was armed. A layout excursion that returns to that exact coordinate has + /// retired no Python opcode, so the operand stack it left is still the + /// operand stack it comes back to and the saved boxes are restored verbatim + /// — the shadow reseed cannot reconstruct them, because mid-expression the + /// virtualizable's stack region holds the NULLs the in-flight opcode's + /// `popvalue_maybe_none` wrote. `None` outside a region. + /// + /// There is nothing to snapshot upstream: `pyjitpl.py:1892` + /// `MIFrame.run_one_step` steps a live frame whose `registers_r` survive + /// the step. This mirror is instead reconstructed from source pcs, and + /// that reconstruction is exactly what an excursion can lose. + pub vstack_reorder_saved: Option<(u32, usize, Vec)>, /// The py_pc the walk's floor lookup reports while it is inside an /// out-of-line exception LANDING block — the unwind bookkeeping the /// codewriter emits per catch site, after the whole body, which then jumps diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs index 2ae807bbca5..0a6803331d2 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs @@ -441,6 +441,7 @@ fn read_ref_reg_concrete_returns_slot_matching_symbolic_read() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -663,6 +664,7 @@ fn getfield_vable_with_none_obj_surfaces_vable_box_not_seeded() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -720,6 +722,7 @@ fn setfield_vable_with_none_obj_surfaces_vable_box_not_seeded() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -795,6 +798,7 @@ fn array_vable_handlers_with_none_obj_surface_vable_box_not_seeded() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -1047,6 +1051,7 @@ fn drive_int_add_jump_if_ovf( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -1205,6 +1210,7 @@ fn drive_alloc_with_descr( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -1399,6 +1405,7 @@ fn run_hint_step_with_descrs( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -1908,6 +1915,7 @@ fn switch_id_hit_jumps_to_matching_target() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -1966,6 +1974,7 @@ fn switch_id_miss_falls_through() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2023,6 +2032,7 @@ fn switch_id_requires_concrete_int_value() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2089,6 +2099,7 @@ fn goto_if_not_truthy_records_guard_true_and_falls_through() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2147,6 +2158,7 @@ fn goto_if_not_falsy_records_guard_false_and_jumps() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2204,6 +2216,7 @@ fn goto_if_not_requires_concrete_int_value() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2470,6 +2483,7 @@ fn inline_call_recursion_writes_subreturn_into_caller_dst_register() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2639,6 +2653,7 @@ fn inline_call_r_i_writes_int_subreturn_into_caller_int_bank() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2756,6 +2771,7 @@ fn inline_call_ir_r_populates_callee_int_and_ref_banks() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2868,6 +2884,7 @@ fn inline_call_irf_r_populates_all_three_kind_banks() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -2969,6 +2986,7 @@ fn inline_call_ir_int_arity_overflow_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3066,6 +3084,7 @@ fn inline_call_recursion_propagates_subraise_from_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3145,6 +3164,7 @@ fn inline_call_with_unresolvable_descr_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3203,6 +3223,7 @@ fn inline_call_with_missing_sub_jitcode_lookup_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3257,6 +3278,7 @@ fn step_through_live_opcode_advances_by_offset_size() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3320,6 +3342,7 @@ fn step_through_ref_return_records_finish_with_descr_and_correct_arg() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3381,6 +3404,7 @@ fn ref_return_with_out_of_range_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3443,6 +3467,7 @@ fn raise_with_unwritten_register_surfaces_register_read_unbound() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3506,6 +3531,7 @@ fn step_through_int_return_records_finish_with_int_descr() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3586,6 +3612,7 @@ fn step_through_int_return_subwalk_surfaces_subreturn_some() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3654,6 +3681,7 @@ fn step_through_void_return_stashes_void_finish_payload() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3723,6 +3751,7 @@ fn step_through_void_return_subwalk_surfaces_subreturn_none() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3780,6 +3809,7 @@ fn raise_with_out_of_range_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3840,6 +3870,7 @@ fn step_through_goto_jumps_to_label_target() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -3900,6 +3931,7 @@ fn step_through_goto_handles_high_byte_of_label() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4008,6 +4040,7 @@ fn step_through_catch_exception_with_active_exception_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4063,6 +4096,7 @@ fn step_through_catch_exception_advances_past_label_operand() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4132,6 +4166,7 @@ fn step_through_raise_records_outermost_finish_and_terminates() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4215,6 +4250,7 @@ fn top_level_raise_settles_the_vable_token() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4322,6 +4358,7 @@ fn raise_r_emits_guard_class_when_concrete_exc_pinned_in_shadow() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4423,6 +4460,7 @@ fn step_through_reraise_at_top_level_records_outermost_finish() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4501,6 +4539,7 @@ fn step_through_reraise_without_last_exc_value_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4556,6 +4595,7 @@ fn raise_at_top_level_populates_last_exc_value_before_finish() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4675,6 +4715,7 @@ fn inline_call_subraise_jumps_to_caller_catch_exception_target() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4797,6 +4838,7 @@ fn inline_call_subraise_without_caller_catch_bubbles_up_in_subwalk() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4867,6 +4909,7 @@ fn step_through_int_copy_advances_past_operand_bytes() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4934,6 +4977,7 @@ fn int_copy_writes_src_value_into_dst_register() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -4992,6 +5036,7 @@ fn int_copy_with_out_of_range_dst_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5049,6 +5094,7 @@ fn int_copy_with_out_of_range_src_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5126,6 +5172,7 @@ fn step_through_ref_copy_advances_past_operand_bytes() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5191,6 +5238,7 @@ fn ref_copy_writes_src_value_into_dst_register() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5247,6 +5295,7 @@ fn ref_copy_with_out_of_range_dst_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5302,6 +5351,7 @@ fn ref_copy_with_out_of_range_src_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5368,6 +5418,7 @@ fn drive_int_binop(opname: &str, expected_opcode: majit_ir::OpCode) { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5594,6 +5645,7 @@ fn drive_int_between( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5732,6 +5784,7 @@ fn drive_float_binop(opname: &str, expected_opcode: majit_ir::OpCode) { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5823,6 +5876,7 @@ fn drive_float_unop(opname: &str, expected_opcode: majit_ir::OpCode) { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -5902,6 +5956,7 @@ fn drive_int_unop(opname: &str, expected_opcode: majit_ir::OpCode) { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6001,6 +6056,7 @@ fn drive_ptr_compare(opname: &str, expected_opcode: majit_ir::OpCode) { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6173,6 +6229,7 @@ fn run_float_step( vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6368,6 +6425,7 @@ fn float_add_with_out_of_range_src_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6424,6 +6482,7 @@ fn int_add_with_out_of_range_src_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6482,6 +6541,7 @@ fn int_add_with_out_of_range_dst_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6548,6 +6608,7 @@ fn unsupported_opname_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6605,6 +6666,7 @@ fn ptr_nonzero_records_ptrne_with_box_and_null() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6755,6 +6817,7 @@ fn abort_result_r_is_pure_pc_advance() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6822,6 +6885,7 @@ fn ref_guard_value_records_guardvalue_with_concrete_constant() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6907,6 +6971,7 @@ fn int_guard_value_records_guardvalue_with_concrete_constant() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -6993,6 +7058,7 @@ fn ref_guard_value_on_const_records_nothing() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7089,6 +7155,7 @@ fn step_through_residual_call_r_r_records_callr_with_descr_and_args() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7259,6 +7326,7 @@ fn residual_call_r_r_with_elidable_cannot_raise_records_callpurer_no_guard() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7346,6 +7414,7 @@ fn authoritative_walker_executes_may_force_call_and_stamps_result() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7405,6 +7474,7 @@ fn non_authoritative_walker_does_not_execute_may_force_call() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7478,6 +7548,7 @@ fn authoritative_walker_transcribes_may_force_raise_to_last_exc() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7581,6 +7652,7 @@ fn may_force_with_active_vable_executes_and_clears_token() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7683,6 +7755,7 @@ fn may_force_vable_escape_surfaces_typed_abort() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7760,6 +7833,7 @@ fn residual_call_r_r_with_not_in_trace_oopspec_returns_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7824,6 +7898,7 @@ fn residual_call_r_r_with_jit_force_virtual_oopspec_returns_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7882,6 +7957,7 @@ fn residual_call_r_r_with_elidable_can_raise_records_callpurer_plus_guard() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -7952,6 +8028,7 @@ fn residual_call_r_r_with_cannot_raise_records_callr_no_guard() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8024,6 +8101,7 @@ fn residual_call_r_r_writes_recorder_result_into_dst_register() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8116,6 +8194,7 @@ fn residual_call_r_r_can_raise_writes_dst_before_guard_no_exception() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8197,6 +8276,7 @@ fn residual_call_ir_r_can_raise_writes_dst_before_guard_no_exception() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8277,6 +8357,7 @@ fn residual_call_r_r_with_out_of_range_dst_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8337,6 +8418,7 @@ fn residual_call_r_r_with_descr_index_out_of_range_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8435,6 +8517,7 @@ fn step_through_residual_call_r_i_records_calli_with_int_dst_writeback() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8532,6 +8615,7 @@ fn residual_call_r_i_with_elidable_cannot_raise_records_callpurei_no_guard() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8637,6 +8721,7 @@ fn step_through_residual_call_ir_r_records_callr_with_int_and_ref_args() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8774,6 +8859,7 @@ fn residual_call_ir_r_permutes_argboxes_per_arg_types_abi() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8847,6 +8933,7 @@ fn residual_call_descr_not_call_descr_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8906,6 +8993,7 @@ fn residual_call_r_r_with_out_of_range_arg_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -8989,6 +9077,7 @@ fn walk_return_value_helper_terminates_at_first_ref_return() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9103,6 +9192,7 @@ fn walk_pop_top_helper_terminates_with_recorded_ops() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9232,6 +9322,7 @@ fn helper_descent_defers_the_limit_check_to_the_enclosing_frame() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9322,6 +9413,7 @@ fn inline_call_with_more_args_than_callee_regs_surfaces_arity_mismatch() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9425,6 +9517,7 @@ fn inline_call_r_v_accepts_void_returning_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9507,6 +9600,7 @@ fn inline_call_r_v_rejects_non_void_returning_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9592,6 +9686,7 @@ fn inline_call_ir_v_accepts_void_returning_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9675,6 +9770,7 @@ fn inline_call_ir_v_rejects_non_void_returning_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9763,6 +9859,7 @@ fn inline_call_irf_v_accepts_void_returning_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9849,6 +9946,7 @@ fn inline_call_irf_v_rejects_non_void_returning_callee() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -9924,6 +10022,7 @@ fn getfield_gc_i_cache_miss_records_op_and_writes_dst() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10020,6 +10119,7 @@ fn getfield_gc_i_cache_hit_returns_cached_box_without_recording() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10094,6 +10194,7 @@ fn getfield_gc_r_cache_miss_records_op_and_writes_ref_dst() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10156,6 +10257,7 @@ fn getfield_gc_with_out_of_range_obj_register_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10230,6 +10332,7 @@ fn getfield_vable_i_routes_through_metainterp_and_writes_dst() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10324,6 +10427,7 @@ fn setfield_vable_i_routes_through_metainterp_records_setfield_gc_fallback() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10408,6 +10512,7 @@ fn setfield_gc_i_redundant_write_skips_recording() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10470,6 +10575,7 @@ fn setfield_gc_i_fresh_write_records_op_and_caches_value() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10558,6 +10664,7 @@ fn setfield_gc_r_records_setfieldgc_with_ref_valuebox() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10629,6 +10736,7 @@ fn getarrayitem_gc_r_cache_miss_records_op_and_writes_dst() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10774,6 +10882,7 @@ fn getarrayitem_gc_pure_const_operands_fold_without_recording_or_counting() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10852,6 +10961,7 @@ fn getarrayitem_gc_r_cache_hit_returns_cached_box() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -10922,6 +11032,7 @@ fn setarrayitem_gc_r_records_setarrayitemgc_with_three_args() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -11241,6 +11352,7 @@ fn walk_undecodable_byte_surfaces_typed_error() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -11320,6 +11432,7 @@ fn jit_merge_point_first_visit_continues_then_closes_loop() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -11409,6 +11522,7 @@ fn loop_header_stamps_seen_flag() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -11477,6 +11591,7 @@ fn jit_merge_point_int_form_resolves_jdindex_from_the_int_bank() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -11541,6 +11656,7 @@ fn jit_merge_point_unresolved_green_key_fails_loud() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -11883,6 +11999,7 @@ fn int_scratch_move_carries_the_concrete_shadow_to_the_destination() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -12078,6 +12195,7 @@ fn walker_folds_a_float_result_pure_call_from_the_float_return_register() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, @@ -12178,6 +12296,7 @@ fn mayforce_null_ref_arg_exempts_the_unread_load_global_namespace() { vstack_valid: false, vstack_last_ref: OpRef::NONE, vstack_reorder_ceiling: u32::MAX, + vstack_reorder_saved: None, vstack_handler_landing_py: None, live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs index 7199190dc65..1d604a617c4 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs @@ -446,9 +446,12 @@ pub(crate) fn reconcile_vstack_at_boundary( // instead of running unprotected. if ctx.vstack_reorder_ceiling != u32::MAX && new_pypc > ctx.vstack_reorder_ceiling { ctx.vstack_reorder_ceiling = u32::MAX; + ctx.vstack_reorder_saved = None; } if !cfg_successor && ctx.vstack_reorder_ceiling == u32::MAX { ctx.vstack_reorder_ceiling = (new_pypc as usize).max(prev_pypc) as u32; + ctx.vstack_reorder_saved = + Some((prev_pypc as u32, ctx.vstack_depth, ctx.vstack_boxes.clone())); } let in_reorder_region = ctx.vstack_reorder_ceiling != u32::MAX; let (fallthrough_depth, branch_depth) = @@ -485,6 +488,37 @@ pub(crate) fn reconcile_vstack_at_boundary( // below. let layout_only_boundary = new_depth != fallthrough_depth && new_depth != branch_depth; + // The excursion came back to the coordinate it left. A `LOAD_ATTR` + + // `CALL` pair puts the walk here: the CALL's lowering re-enters the + // LOAD_ATTR source segment, so the floor lookup reports the earlier py_pc, + // and the return boundary reports the CALL's again at the same depth. No + // Python opcode retired in between — the stack the walk left is the stack + // it resumes with — so the saved boxes, not a reseed, are this boundary's + // reconcile. The reseed cannot serve this shape: the operands the + // in-flight opcode already popped read NULL in the virtualizable shadow, + // so `reseed_vstack_from_shadow` rejects them and the callable slides down + // into the `self_or_null` slot with the TOS left a hole. The restore + // replaces only the reconcile; the shadow-backed hole-fill below still + // runs over the result. + // + // The excursion itself is an artifact of walking a flattened jitcode: + // `pyopcode.py:1037` `LOAD_ATTR` pops and pushes the live value stack in + // place, so the interpreter has no walk position to leave and return to, + // and nothing here to mirror. + let returned_to_arm_point = matches!( + &ctx.vstack_reorder_saved, + Some((pc, depth, _)) if *pc == new_pypc && *depth == new_depth + ); + let restored = in_reorder_region && returned_to_arm_point; + if restored { + let (_, _, boxes) = ctx + .vstack_reorder_saved + .take() + .expect("checked by returned_to_arm_point"); + ctx.vstack_boxes = boxes; + ctx.vstack_reorder_ceiling = u32::MAX; + } + // PER-OP RECONCILE. In the SEQUENTIAL case the previous opcode's stack // effect explains the depth change: a producer (`ResultToTos`) lands its // result box (`vstack_last_ref`) on the new TOS; a pop / side-store just @@ -494,12 +528,19 @@ pub(crate) fn reconcile_vstack_at_boundary( // in the walk register bank, never written through to the portal array). // Inside the out-of-order permutation region the per-op replay is invalid; // reseed from the shadow (same shape as `ShadowReseed`). - let effective_class = if in_reorder_region { + let effective_class = if in_reorder_region && !restored { VstackOpClass::ShadowReseed } else { class }; match effective_class { + // The saved boxes ARE the reconcile for this boundary; the previous + // opcode's effect belongs to the excursion, not to the arm point. The + // shadow-backed hole-fill below still runs: a slot the arm point + // already carried as a hole is no more recoverable from the saved + // boxes than from a reseed, and skipping the fill left the operand + // `LOAD_GLOBAL` pushed unsourced in the resume image. + _ if restored => {} // A layout-only boundary (the observed depth matches neither real // successor of the previous opcode) means the per-op effect cannot // explain this transition; preserve the surviving slots instead of diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 9e3046e66d5..0c34fc821c6 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -14091,7 +14091,7 @@ while i < 40: } /// rclass.py:1133-1137 `ll_issubclass(subcls, cls)` parity. After - /// `set_gc_allocator` runs `freeze_types`, the materialized + /// `install_gc_standalone` runs `freeze_types`, the materialized /// `(subclassrange_min, subclassrange_max)` for each registered /// PyType must satisfy `int_between(cls.min, subcls.min, cls.max)` /// for every (cls, subcls) pair where `subcls` Python-inherits from @@ -14105,7 +14105,7 @@ while i < 40: /// `LIST_TYPE`) are disjoint. #[test] fn test_subclass_range_preorder_bounds() { - // Force JIT_DRIVER initialization so set_gc_allocator runs and + // Force JIT_DRIVER initialization so init_gc_subsystem runs and // installs the active subclass_range hook. let _ = driver_pair(); diff --git a/pyre/pyre-jit/tests/gc_stress.rs b/pyre/pyre-jit/tests/gc_stress.rs index a72281b9580..6ff2b3c8f31 100644 --- a/pyre/pyre-jit/tests/gc_stress.rs +++ b/pyre/pyre-jit/tests/gc_stress.rs @@ -79,9 +79,11 @@ fn run_harness(program: &str, name: &str, vacuity_label: &str) -> Result<(), Str eval_with_jit(&mut frame).map_err(|e| format!("execution error: {}", e.message))?; // Non-vacuity: the stable instance allocator hook is installed by the - // `JIT_DRIVER` initializer (`driver_pair` -> `set_gc_allocator`). If it is - // live now, allocations routed through the managed heap rather than the - // leaking Box fallback, so the survival checks were meaningful. + // `JIT_DRIVER` initializer (`driver_pair` -> `init_gc_subsystem` -> + // `install_gc_standalone`, which registers the hooks against the `gc_sync` + // singleton and stores no per-thread box). If it is live now, allocations + // routed through the managed heap rather than the leaking Box fallback, so + // the survival checks were meaningful. let probe = pyre_object::try_gc_alloc_stable( pyre_object::W_OBJECT_OBJECT_GC_TYPE_ID, pyre_object::W_OBJECT_OBJECT_SIZE,