diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index 876724bb8aa..4a34cf5d512 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -4698,21 +4698,13 @@ fn build_known_values_set(inputargs: &[InputArg], ops: &[Op]) -> IndexSet { known } -fn build_force_token_set(inputargs: &[InputArg], ops: &[Op]) -> IndexSet { - // FORCE_TOKEN (resoperation.py:1090 'FORCE_TOKEN/0/r') yields the raw - // jitframe handle. Its Ref result is the frame pointer, not a heap GCREF, - // so it is excluded from the ref-root slots the GC traces and relocates. - let mut force_tokens = IndexSet::new(); - for (op_idx, op) in ops.iter().enumerate() { - if op.pos.get().is_none() { - continue; - } - if op.opcode == OpCode::ForceToken { - let result_var = op_var_index(op, op_idx, inputargs.len()) as u32; - force_tokens.insert(result_var); - } - } - force_tokens +fn build_force_token_set(_inputargs: &[InputArg], _ops: &[Op]) -> IndexSet { + // FORCE_TOKEN is a GCREF to the active JITFRAME + // (`virtualizable.py:315-318`, `resoperation.py:1090`). Keep its in-frame + // copies in the ordinary Ref root set so moving collectors update them. + // The empty compatibility set leaves the existing exit-layout plumbing in + // place while giving FORCE_TOKEN the same treatment as every other Ref. + IndexSet::new() } /// Auxiliary LABEL type overrides on top of `OpTypeIndex`. @@ -8753,9 +8745,10 @@ impl CraneliftBackend { // overwrite by index. That disjoint home is why non-refs (incl. // inputargs) are safe to demote: the ref-inputarg staleness argument // does not apply because the home is re-seeded on every LABEL entry - // edge (preamble fall-through and loader re-entry). Floats, SIMD - // lanes, and force tokens are excluded — an 8-byte home cannot hold a - // vector and float typing must round-trip through resume. + // edge (preamble fall-through and loader re-entry). Floats and SIMD + // lanes are excluded — an 8-byte home cannot hold a vector and float + // typing must round-trip through resume. FORCE_TOKEN is a Ref and uses + // the forwarded ref-root home. if !loop_phi_keep_by_label.is_empty() { let input_idxs: indexmap::IndexSet = inputargs.iter().map(|ia| ia.index).collect(); loop_phi_keep_by_label.retain(|&label_idx, keep| { diff --git a/majit/majit-metainterp/src/jitcode/assembler.rs b/majit/majit-metainterp/src/jitcode/assembler.rs index 8f017ce239c..2d900426717 100644 --- a/majit/majit-metainterp/src/jitcode/assembler.rs +++ b/majit/majit-metainterp/src/jitcode/assembler.rs @@ -624,6 +624,28 @@ impl JitCodeBuilder { headerless: bool, fields: &[(usize, bool, &str)], ) { + // Every emit site re-registers the layout it accesses, so the same few + // type_ids arrive hundreds of times. When the spec already lists an + // offset for each incoming field the merge below pushes nothing, and + // the re-sort/re-index that follow it are no-ops on an already sorted, + // already indexed vector — the branch never touches `size`, + // `is_gc_managed` or `headerless`, so returning here is exactly + // equivalent and skips building `new_fields` (one owned String per + // field) only to discard it. + if self + .struct_size_specs + .get(&type_id) + .is_some_and(|existing| { + fields.iter().all(|&(offset, _, _)| { + existing + .all_fielddescrs + .iter() + .any(|ef| ef.offset == offset) + }) + }) + { + return; + } let new_fields = Self::field_specs_from_layout(fields); // Merge into existing spec if present — each getfield/setfield // site registers only the field it accesses, so the complete @@ -760,8 +782,7 @@ impl JitCodeBuilder { type_id: u64, field_name: &str, ) -> u16 { - let parent = self.struct_size_specs.get(&type_id).cloned(); - let Some(parent_spec) = parent.as_ref() else { + let Some(parent_spec) = self.struct_size_specs.get(&type_id) else { return self.add_scalar_field_descr(offset, field_type); }; let (field_flag, is_field_signed) = match field_type { @@ -793,6 +814,22 @@ impl JitCodeBuilder { field_slot_in(&parent_spec.all_fielddescrs, field_name, offset) .map(|idx| (idx, parent_spec.all_fielddescrs[idx].name.clone())) .unwrap_or((0, String::new())); + // Carry the scalars only. `patch_field_descr_parents`, called + // unconditionally from `try_finish` after the decline early-return, + // replaces this snapshot with `struct_size_specs`' final merged spec + // for the same `type_id` — and entries there are only inserted or + // merged, never removed, so a type_id present now is present then. + // `type_id` is therefore the only load-bearing part; deep-copying + // `all_fielddescrs` (a String per field) once per field-descr mint + // would build a table that is overwritten before anything reads it. + let parent = Some(BhSizeSpec { + size: parent_spec.size, + type_id: parent_spec.type_id, + vtable: parent_spec.vtable, + is_gc_managed: parent_spec.is_gc_managed, + headerless: parent_spec.headerless, + all_fielddescrs: Vec::new(), + }); self.add_bh_descr(CanonicalBhDescr::Field { offset, field_size: scalar_size(field_type), diff --git a/majit/majit-metainterp/src/jitdriver.rs b/majit/majit-metainterp/src/jitdriver.rs index 16356c97a07..c87db00f829 100644 --- a/majit/majit-metainterp/src/jitdriver.rs +++ b/majit/majit-metainterp/src/jitdriver.rs @@ -2984,6 +2984,13 @@ impl JitDriver { // false now and true at the next visit of this header. if let Some(ctx) = self.meta.trace_ctx() { if attempted { + // The two sibling close sites report a declined + // attempt under the same tally; this one is the + // third. Bump it on the same condition the latch + // uses, so the census counts closes an optimizer + // pass actually rejected and not headers the gate + // above skipped. + crate::mc_diag_bump(50); // bridge_declined_close ctx.note_cross_loop_close_declined(target_key); } ctx.close_greens = None; diff --git a/majit/majit-metainterp/src/jitprof.rs b/majit/majit-metainterp/src/jitprof.rs index 454ef7ad176..ebefbb3bd69 100644 --- a/majit/majit-metainterp/src/jitprof.rs +++ b/majit/majit-metainterp/src/jitprof.rs @@ -21,9 +21,9 @@ use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; #[cfg(not(target_arch = "wasm32"))] -use std::time::Instant; +pub use std::time::Instant; #[cfg(target_arch = "wasm32")] -use wasm_clock::Instant; +pub use wasm_clock::Instant; /// Monotonic substitute for `std::time::Instant` on wasm32-unknown-unknown, /// which has no clock (`Instant::now()` there panics). The profiler only needs @@ -33,7 +33,7 @@ use wasm_clock::Instant; /// `Instant::now()` / `saturating_duration_since` surface the timer uses, so /// the timing code below is platform-agnostic. #[cfg(target_arch = "wasm32")] -mod wasm_clock { +pub mod wasm_clock { use core::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; diff --git a/majit/majit-metainterp/src/pyjitpl.rs b/majit/majit-metainterp/src/pyjitpl.rs index 5290f07781e..f49b52e5c3d 100644 --- a/majit/majit-metainterp/src/pyjitpl.rs +++ b/majit/majit-metainterp/src/pyjitpl.rs @@ -21,6 +21,7 @@ use std::sync::{ atomic::{AtomicU64, Ordering}, }; +use crate::jitprof::Instant; use crate::optimizeopt::optimizer::{Optimizer, PendingBridgeRd}; use majit_backend::{Backend, ExitRecoveryLayout, JitCellToken}; #[cfg(all(feature = "cranelift", not(target_arch = "wasm32")))] @@ -6061,6 +6062,7 @@ impl MetaInterp { Vec, crate::optimizeopt::unroll::ExportedState, )> = None; + let optimize_start = Instant::now(); let optimize_result = if no_unroll { if crate::majit_log_enabled() { eprintln!( @@ -6263,6 +6265,7 @@ impl MetaInterp { optimized_ops } }; + let opt_time = Instant::now().saturating_duration_since(optimize_start); let num_ops_after = optimized_ops.len(); if crate::majit_log_enabled() { eprintln!( @@ -6625,12 +6628,14 @@ impl MetaInterp { compiled_ops.len() ); } + let compile_start = Instant::now(); let compile_result = { let _backend_scope = self.staticdata.profiler.enter_backend(); std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { self.backend.compile_loop(&inputargs, &compiled_ops, &token) })) }; + let compile_time = Instant::now().saturating_duration_since(compile_start); let compile_result = match compile_result { Ok(r) => r, Err(e) => { @@ -6808,8 +6813,8 @@ impl MetaInterp { green_key, num_ops_before, num_ops_after, - std::time::Duration::ZERO, - std::time::Duration::ZERO, + opt_time, + compile_time, ); // warmstate.py:339-348 attach the same compiled token object. self.attach_procedure_with_redirect(green_key, Arc::clone(&token)); @@ -7614,6 +7619,7 @@ impl MetaInterp { // the pre-peel arg set for slots the loop label rebinds. unroll_opt.emit_start_label = false; + let optimize_start = Instant::now(); let optimize_result = unroll_opt.optimize_trace_with_constants_and_inputs_vable( &trace_ops, &mut constants, @@ -7634,6 +7640,7 @@ impl MetaInterp { return false; } }; + let opt_time = Instant::now().saturating_duration_since(optimize_start); // compile.py:384-390: merge loop_info deps first, then deps carried // from the exported start_state. let mut quasi_immutable_deps = std::mem::take(&mut unroll_opt.quasi_immutable_deps); @@ -7809,6 +7816,7 @@ impl MetaInterp { // compile.py:532-546 `debug_start("jit-backend") + // profiler.start_backend() ... try: do_compile_loop ... finally: // ... profiler.end_backend() + debug_stop("jit-backend")`. + let compile_start = Instant::now(); let compile_result = { let _backend_scope = self.staticdata.profiler.enter_backend(); std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -7820,6 +7828,7 @@ impl MetaInterp { ) })) }; + let compile_time = Instant::now().saturating_duration_since(compile_start); let compile_result = match compile_result { Ok(r) => r, Err(payload) => { @@ -7974,8 +7983,8 @@ impl MetaInterp { green_key, num_ops_before, num_combined_ops, - std::time::Duration::ZERO, - std::time::Duration::ZERO, + opt_time, + compile_time, ); self.attach_procedure_with_redirect(green_key, Arc::clone(&token)); self.stats.loops_compiled += 1; @@ -8569,6 +8578,7 @@ impl MetaInterp { // InvalidLoop during optimization should abort the trace, not crash // the process. Matches compile_loop. + let optimize_start = Instant::now(); let optimize_result = optimizer.optimize_with_constants_and_inputs_oprc( // `trace.ops` are the canonical `Rc`, so `input_ops` // seeds identity directly from them. @@ -8599,6 +8609,7 @@ impl MetaInterp { return Err(SwitchToBlackhole::giveup()); } }; + let opt_time = Instant::now().saturating_duration_since(optimize_start); // RPython optimizer.py:552-556 (flush=True): Finish/Jump is sent // through passes inside propagate_all_forward and ends up in // new_operations naturally — no restoration needed. @@ -8731,11 +8742,13 @@ impl MetaInterp { // compile.py:532-546 `debug_start("jit-backend") + // profiler.start_backend() ... try: do_compile_loop ... finally: // ... profiler.end_backend() + debug_stop("jit-backend")`. + let compile_start = Instant::now(); let compile_loop_result = { let _backend_guard = self.staticdata.profiler.enter_backend(); self.backend .compile_loop(&inputargs, &optimized_ops, &token) }; + let compile_time = Instant::now().saturating_duration_since(compile_start); match compile_loop_result { Ok(_) => { self.last_compiled_artifact_invalidation_flag = Some(token.invalidation_flag()); @@ -8857,6 +8870,13 @@ impl MetaInterp { }, ); } + self.warm_state.log_compile( + green_key, + num_ops_before, + num_ops_after, + opt_time, + compile_time, + ); self.attach_procedure_with_redirect(green_key, Arc::clone(&token)); self.stats.loops_compiled += 1; // `cpu.tracker.total_compiled_loops` is bumped inside @@ -8990,6 +9010,7 @@ impl MetaInterp { optimizer.snapshot_vref_boxes = snapshot_vref_map; optimizer.snapshot_frame_pcs = snapshot_pc_map; + let optimize_start = Instant::now(); let optimize_result = optimizer.optimize_with_constants_and_inputs_oprc( // Canonical `Rc`; `input_ops` seeds identity from them. &trace.ops, @@ -9013,6 +9034,7 @@ impl MetaInterp { return None; } }; + let opt_time = Instant::now().saturating_duration_since(optimize_start); // optimizer.py:557 self.resumedata_memo.update_counters(profiler) optimizer.update_counters(&self.staticdata.profiler); @@ -9106,6 +9128,7 @@ impl MetaInterp { // compile.py:532-546 `debug_start("jit-backend") + // profiler.start_backend() ... try: do_compile_loop ... finally: // ... profiler.end_backend() + debug_stop("jit-backend")`. + let compile_start = Instant::now(); let compile_loop_result = { let _backend_guard = self.staticdata.profiler.enter_backend(); self.backend.compile_loop( @@ -9115,6 +9138,7 @@ impl MetaInterp { .expect("JitCellToken must stay uniquely owned until backend compile"), ) }; + let compile_time = Instant::now().saturating_duration_since(compile_start); match compile_loop_result { Ok(_) => { self.assign_guard_hashes(token.as_ref()); @@ -9198,6 +9222,13 @@ impl MetaInterp { next_global_opref, }, ); + self.warm_state.log_compile( + green_key, + num_ops_before, + num_ops_after, + opt_time, + compile_time, + ); self.stats.loops_compiled += 1; // `cpu.tracker.total_compiled_loops` is bumped inside // `CompiledLoopToken::new` (model.py:297 parity). @@ -11381,6 +11412,7 @@ impl MetaInterp { // constant pool merge. Const objects flow via rd_consts + fresh // decode (resume.py:1245-1282). let retrace_limit = self.warm_state.retrace_limit(); + let optimize_start = Instant::now(); let bridge_optimize_result = { let compiled = self.compiled_loops.get_mut(&green_key).unwrap(); optimizer.optimize_bridge( @@ -11419,6 +11451,7 @@ impl MetaInterp { return false; } }; + let opt_time = Instant::now().saturating_duration_since(optimize_start); // optimizer.py:557 self.resumedata_memo.update_counters(profiler) optimizer.update_counters(&self.staticdata.profiler); // RPython-orthodox: unroll.py replay uses Const args directly; @@ -11504,6 +11537,7 @@ impl MetaInterp { // compile.py:532-546 `debug_start("jit-backend") + // profiler.start_backend() ... try: do_compile_loop ... finally: // ... profiler.end_backend() + debug_stop("jit-backend")`. + let compile_start = Instant::now(); let compile_result = { let _backend_scope = self.staticdata.profiler.enter_backend(); std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -11511,6 +11545,7 @@ impl MetaInterp { .compile_loop(bridge_inputargs, &optimized_ops, &token) })) }; + let compile_time = Instant::now().saturating_duration_since(compile_start); let compile_result = match compile_result { Ok(r) => r, Err(payload) => { @@ -11636,6 +11671,13 @@ impl MetaInterp { next_global_opref, }, ); + self.warm_state.log_compile( + original_green_key, + bridge_ops.len(), + num_optimized_ops, + opt_time, + compile_time, + ); self.attach_procedure_with_redirect(original_green_key, Arc::clone(&token)); self.stats.loops_compiled += 1; // `cpu.tracker.total_compiled_loops` is bumped inside diff --git a/majit/majit-metainterp/src/virtualizable.rs b/majit/majit-metainterp/src/virtualizable.rs index c42ed63ce82..4229ff9729a 100644 --- a/majit/majit-metainterp/src/virtualizable.rs +++ b/majit/majit-metainterp/src/virtualizable.rs @@ -19,19 +19,17 @@ use std::sync::{Arc, Weak}; use majit_ir::{DescrRef, Type, descr::descr_identity}; -/// Sentinel value for TOKEN_TRACING_RESCALL. -/// -/// When the token equals this value, it means JIT tracing is active and -/// a residual call is in progress. If the callee touches the virtualizable, -/// it will force the token and clear it. -/// -/// Any non-zero, non-RESCALL value is an active JIT frame pointer. -pub const TOKEN_TRACING_RESCALL: u64 = u64::MAX; +/// `virtualizable.py:330 TOKEN_TRACING_RESCALL`: the GCREF address of the +/// prebuilt `JITFRAME_DUMMY` object shared with virtual references. +#[inline] +pub fn token_tracing_rescall() -> u64 { + crate::virtualref::token_tracing_rescall() as usize as u64 +} /// Token states for virtualizable objects. /// /// TOKEN_NONE (0): not in JIT. -/// TOKEN_TRACING_RESCALL (u64::MAX): tracing + residual call in progress. +/// TOKEN_TRACING_RESCALL (prebuilt GCREF): tracing + residual call in progress. /// Any other non-zero value: active JIT frame pointer (FORCE_TOKEN). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VableToken { @@ -49,7 +47,7 @@ impl VableToken { pub fn from_raw(raw: u64) -> Self { match raw { 0 => VableToken::None, - TOKEN_TRACING_RESCALL => VableToken::TracingRescall, + other if other == token_tracing_rescall() => VableToken::TracingRescall, other => VableToken::Active(other), } } @@ -58,7 +56,7 @@ impl VableToken { pub fn to_raw(self) -> u64 { match self { VableToken::None => 0, - VableToken::TracingRescall => TOKEN_TRACING_RESCALL, + VableToken::TracingRescall => token_tracing_rescall(), VableToken::Active(ptr) => ptr, } } @@ -736,7 +734,7 @@ impl VirtualizableInfo { // The all-ones sentinel truncates to `usize::MAX` at that width. let token_ptr = obj_ptr.add(self.token_offset) as *mut usize; assert_eq!(*token_ptr, 0, "token should be NONE before residual call"); - *token_ptr = TOKEN_TRACING_RESCALL as usize; + *token_ptr = token_tracing_rescall() as usize; } } @@ -757,7 +755,7 @@ impl VirtualizableInfo { let token_ptr = obj_ptr.add(self.token_offset) as *mut usize; if *token_ptr != 0 { // Not forced — still TOKEN_TRACING_RESCALL - assert_eq!(*token_ptr, TOKEN_TRACING_RESCALL as usize); + assert_eq!(*token_ptr, token_tracing_rescall() as usize); *token_ptr = 0; // Clear back to TOKEN_NONE false } else { @@ -782,7 +780,7 @@ impl VirtualizableInfo { unsafe { let token_ptr = obj_ptr.add(self.token_offset) as *mut usize; let token = *token_ptr; - if token == TOKEN_TRACING_RESCALL as usize { + if token == token_tracing_rescall() as usize { // During tracing — just clear the marker *token_ptr = 0; } else if token != 0 { @@ -804,15 +802,7 @@ impl VirtualizableInfo { unsafe { let token_ptr = obj_ptr.add(self.token_offset) as *const usize; let raw = *token_ptr; - // The all-ones sentinel is stored width-truncated (`usize::MAX` - // on wasm32); widen it back to the canonical `u64::MAX` so - // `from_raw` matches. - let raw_u64 = if raw == TOKEN_TRACING_RESCALL as usize { - TOKEN_TRACING_RESCALL - } else { - raw as u64 - }; - VableToken::from_raw(raw_u64) + VableToken::from_raw(raw as u64) } } @@ -827,6 +817,12 @@ impl VirtualizableInfo { unsafe { let token_ptr = obj_ptr.add(self.token_offset) as *mut usize; *token_ptr = token.to_raw() as usize; + if matches!(token, VableToken::Active(_)) { + // Host-side active-token stores have the same generational + // obligation as compiled SETFIELD_GC stores. Unmanaged test + // objects are ignored by the barrier hook. + majit_gc::gc_write_barrier(majit_ir::GcRef(obj_ptr as usize)); + } } } @@ -1564,7 +1560,7 @@ unsafe fn is_token_nonnull(info: &VirtualizableInfo, obj_ptr: *const u8) -> bool /// /// Token semantics: /// - TOKEN_NONE (0): not in JIT, nothing to do. -/// - TOKEN_TRACING_RESCALL (u64::MAX): tracing + residual call, just clear. +/// - TOKEN_TRACING_RESCALL (prebuilt GCREF): tracing + residual call, just clear. /// - Any other non-zero value: active JIT frame pointer. Call `force_fn` /// with the frame pointer, which must clear the token itself. /// @@ -2514,15 +2510,16 @@ mod tests { #[test] fn test_vable_token_roundtrip() { + let tracing_rescall = token_tracing_rescall(); assert_eq!(VableToken::from_raw(0), VableToken::None); assert_eq!( - VableToken::from_raw(TOKEN_TRACING_RESCALL), + VableToken::from_raw(tracing_rescall), VableToken::TracingRescall ); assert_eq!(VableToken::from_raw(0xBEEF), VableToken::Active(0xBEEF)); assert_eq!(VableToken::None.to_raw(), 0); - assert_eq!(VableToken::TracingRescall.to_raw(), TOKEN_TRACING_RESCALL); + assert_eq!(VableToken::TracingRescall.to_raw(), tracing_rescall); assert_eq!(VableToken::Active(0xBEEF).to_raw(), 0xBEEF); } @@ -2577,7 +2574,7 @@ mod tests { let obj_ptr = obj.as_mut_ptr(); unsafe { - *(obj_ptr as *mut u64) = TOKEN_TRACING_RESCALL; + *(obj_ptr as *mut u64) = token_tracing_rescall(); let mut called = false; info.force_now(obj_ptr, |_| { @@ -2656,7 +2653,7 @@ mod tests { let obj_ptr = obj.as_mut_ptr(); unsafe { - *(obj_ptr as *mut u64) = TOKEN_TRACING_RESCALL; + *(obj_ptr as *mut u64) = token_tracing_rescall(); let mut called = false; info.clear_vable_token(obj_ptr, |_| { diff --git a/majit/majit-metainterp/src/virtualref.rs b/majit/majit-metainterp/src/virtualref.rs index 934de8276f9..114f60c4871 100644 --- a/majit/majit-metainterp/src/virtualref.rs +++ b/majit/majit-metainterp/src/virtualref.rs @@ -13,7 +13,7 @@ //! //! Mirrors `rpython/jit/metainterp/virtualref.py`. -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; /// The value [`VREF_GC_TYPE_ID`] holds before `set_vref_gc_type_id` runs. Zero /// is a legitimate id, so the sentinel has to be a value the registry never @@ -68,27 +68,9 @@ pub struct ObjectHeader { /// `make_vref_field_descr` (`Type::Ref` per /// `optimizeopt/virtualize.rs`) agree on the slot type. /// -/// TODO (GC trace). Upstream traces both fields as -/// real GC pointers; pyre traces only `forced` — the type registration -/// in `eval.rs` derives that one entry from `offset_of!(JitVirtualRef, -/// forced)`, so it follows the target's pointer width. The reason -/// `virtual_token` is left out is that every value it ever holds at -/// runtime falls outside the GC heap: -/// - `TOKEN_NONE` — null, safe to walk. -/// - `token_tracing_rescall()` — program-lifetime leaked -/// `Box` (see `allocate_tracing_rescall_dummy` / -/// `TRACING_RESCALL_DUMMY_PTR` below), host-heap allocated and -/// never freed; not a GC-allocated `_dummy` GcStruct. -/// - an active JITFRAME address — `libc::calloc`'d on a host-side -/// pool, not nursery/oldgen. -/// Routing it through `trace_and_update_object` would either be a -/// no-op or trip a poison-address check. The optimizer-side -/// `Type::Ref` is intentionally retained so that -/// `setfield_gc_r` / `getfield_gc_r` ops emit correctly during -/// tracing; only the collector's view of the slot diverges. -/// Convergence path: would require allocating `_dummy` via the GC -/// AND routing JITFRAMEs through GC-managed allocation, both outside -/// the current parity scope. +/// Both fields are traced GC slots. `virtual_token` contains null, the +/// prebuilt `JITFRAME_DUMMY`, or a JITFRAME GCREF, matching +/// `virtualref.py:17-20` and `virtualizable.py:326-330`. #[repr(C)] pub struct JitVirtualRef { /// `('super', rclass.OBJECT)` — typeptr slot at offset 0. @@ -158,10 +140,10 @@ pub use crate::jit::InvalidVirtualRef; /// Returns raw pointer; caller owns the allocation. /// /// `lltype.malloc(self.JIT_VIRTUAL_REF)` is a GC allocation, and it has to be -/// one here too: `forced` is the sole traced slot of the type registered with -/// [`set_vref_gc_type_id`], so once `ExecutionContext.topframeref` holds -/// the vref instead of the frame, this object is the only edge keeping the -/// frame it wraps reachable. A host-heap allocation is invisible to the +/// one here too: `forced` is a traced slot of the type registered with +/// [`set_vref_gc_type_id`], so once `ExecutionContext.topframeref` holds the +/// vref instead of the frame, this edge keeps the frame it wraps reachable. +/// A host-heap allocation is invisible to the /// collector — the root walker's `gc_current_object_address` early-out returns /// an unowned address unchanged — which drops that edge and lets a live frame /// be collected out from under the walk. @@ -213,24 +195,58 @@ pub const TOKEN_NONE: *mut u8 = std::ptr::null_mut(); /// the translated identity word is pointer-sized too. pub const JITFRAME_DUMMY_VTABLE: usize = 0x4A46_444D; // "JFDM" -/// Lazy initialisation of the `_dummy` address. `OnceLock` -/// (instead of `OnceLock<*mut u8>`) so the cell is `Sync` — -/// raw-pointer types are not. -static TRACING_RESCALL_DUMMY_PTR: std::sync::OnceLock = std::sync::OnceLock::new(); +/// Lazy initialisation of the `_dummy` address, as a `usize` because raw +/// pointers are not `Sync`. Zero means "not minted yet"; a null sentinel would +/// collide with `TOKEN_NONE`, so it is not a value this can ever hold. +static TRACING_RESCALL_DUMMY_PTR: AtomicUsize = AtomicUsize::new(0); + +const TRACING_RESCALL_DUMMY_GC_TYPE_ID_UNSET: u32 = u32::MAX; +static TRACING_RESCALL_DUMMY_GC_TYPE_ID: AtomicU32 = + AtomicU32::new(TRACING_RESCALL_DUMMY_GC_TYPE_ID_UNSET); + +/// Publish the registered leaf type used by the prebuilt +/// `virtualizable.py:326-330 JITFRAME_DUMMY` object. +/// +/// Registration comes from `build_gc`, so a second call means a second heap. +/// A sentinel minted in the previous one is no longer part of the live heap — +/// `is_managed_heap_object` would stop recognising it and the traced +/// `virtual_token` / `vable_token` slots would be back to holding an address +/// the collector does not own. Drop it so the next request mints in the heap +/// that is now current. +pub fn set_tracing_rescall_dummy_gc_type_id(type_id: u32) { + TRACING_RESCALL_DUMMY_GC_TYPE_ID.store(type_id, Ordering::Relaxed); + TRACING_RESCALL_DUMMY_PTR.store(0, Ordering::Relaxed); +} /// `virtualizable.py:327 _dummy = lltype.malloc(_DUMMY)` — allocate -/// the singleton dummy `JITFRAME_DUMMY` object whose address serves -/// as the tracing sentinel. Pyre's `Box::into_raw(Box::new(...))` -/// produces a heap-allocated, stable, non-null address; the -/// `Box::leak` semantic (the box is intentionally never freed) -/// matches upstream's `immortal=True`-equivalent lifetime — `_dummy` -/// is allocated once at first use and stays live for the rest of -/// the program. +/// the singleton dummy `JITFRAME_DUMMY` object whose address serves as the +/// tracing sentinel. The object is allocated in the GC old generation with a +/// registered leaf type and held by a program-lifetime root, giving it the +/// prebuilt lifetime of `_dummy` while remaining a legal GCREF. fn allocate_tracing_rescall_dummy() -> *mut u8 { - let header = Box::new(ObjectHeader { + let value = ObjectHeader { typeptr: JITFRAME_DUMMY_VTABLE, - }); - Box::into_raw(header) as *mut u8 + }; + let type_id = TRACING_RESCALL_DUMMY_GC_TYPE_ID.load(Ordering::Relaxed); + if type_id == TRACING_RESCALL_DUMMY_GC_TYPE_ID_UNSET { + // The leaf type is registered by the same setup that installs a + // collector, so an unset id means there is no managed heap to mint the + // prebuilt object in — the token protocol is being driven without one, + // as the walker and dispatch unit tests do. A host allocation keeps the + // address stable and unique for the process while staying outside the + // managed heap, where `is_managed_heap_object` rejects it before any + // tracing path reads its header. + return Box::into_raw(Box::new(value)) as *mut u8; + } + let dummy = majit_gc::alloc_oldgen_typed(type_id, std::mem::size_of::()); + assert!(!dummy.is_null(), "JITFRAME_DUMMY old-gen allocation failed"); + unsafe { std::ptr::write(dummy.0 as *mut ObjectHeader, value) }; + + // `_dummy` is prebuilt and immortal upstream. A leaked root slot gives the + // collector the same lifetime; old-gen makes its identity stable. + let root = Box::into_raw(Box::new(dummy)); + unsafe { majit_gc::gc_add_root(root) }; + dummy.0 as *mut u8 } /// Token value used during tracing when a residual call is in progress. @@ -238,21 +254,23 @@ fn allocate_tracing_rescall_dummy() -> *mut u8 { /// ```python /// TOKEN_TRACING_RESCALL = lltype.cast_opaque_ptr(llmemory.GCREF, _dummy) /// ``` -/// Pyre returns the address of a real heap-allocated `ObjectHeader` -/// initialised lazily on first call; subsequent calls return the -/// same address (program-lifetime immortal). -/// -/// TODO (GC registration). Upstream's `_dummy` -/// is a real GcStruct that the collector knows about; pyre's leaked -/// `Box` is host-allocated memory the GC has no -/// record of. The adaptation is internally consistent because -/// `virtual_token` is not GC-traced either (see `JitVirtualRef` -/// doc-comment); if the slot ever becomes GC-traced, the `_dummy` -/// allocation must move to the GC heap and a JITFRAME_DUMMY type -/// must be registered with the collector. +/// The returned address is a registered, rooted GC leaf, and stays the same for +/// as long as the heap it was minted in is the current one. #[inline] -fn token_tracing_rescall() -> *mut u8 { - *TRACING_RESCALL_DUMMY_PTR.get_or_init(|| allocate_tracing_rescall_dummy() as usize) as *mut u8 +pub fn token_tracing_rescall() -> *mut u8 { + let minted = TRACING_RESCALL_DUMMY_PTR.load(Ordering::Relaxed); + if minted != 0 { + return minted as *mut u8; + } + let fresh = allocate_tracing_rescall_dummy() as usize; + // Racing minters both produce a valid sentinel, but the token protocol + // compares tokens by address, so exactly one may be published. The loser's + // object is immortal either way — upstream's `_dummy` is prebuilt. + match TRACING_RESCALL_DUMMY_PTR.compare_exchange(0, fresh, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => fresh as *mut u8, + Err(published) => published as *mut u8, + } } /// Virtual reference state for a single reference. diff --git a/pyre/pyre-interpreter/src/module/_pickle/pickler.rs b/pyre/pyre-interpreter/src/module/_pickle/pickler.rs index e9b3ac30bd7..6d279d6ff91 100644 --- a/pyre/pyre-interpreter/src/module/_pickle/pickler.rs +++ b/pyre/pyre-interpreter/src/module/_pickle/pickler.rs @@ -46,6 +46,13 @@ fn pickler_write_barrier(obj: PyObjectRef) { pub struct W_Pickler { /// Output file (has a `write` method). w_file: PyObjectRef, + /// Bound `file.write`, resolved once by `__init__` and reused for every + /// write. `interp_pickle.py` resolves it at `:555-560` only to validate, + /// discards the result, and re-resolves per write in `_Framer.file_write` + /// (`:353`); measured on 3.14.5, rebinding `file.write` after construction + /// is *not* observed by a later `dump()`, and `pickle.py:465` captures the + /// callable the same way. Reusing the resolution is what matches. + w_write: PyObjectRef, proto: i64, bin: bool, framing: bool, @@ -230,21 +237,24 @@ fn add_reduce_note(err: PyError, w_obj_slot: Option, role: &str) -> PyErr /// boundary, `write_large_bytes`, and the end flush — and those callers pin the /// objects they still need across the `file.write` (arbitrary Python). /// -/// `file_slot` is the shadow-stack slot of the destination file (pinned by the -/// caller for the whole dump). When it is `None` (the `dumps` path) nothing is +/// `file_slot` is the shadow-stack slot of the destination file and +/// `write_slot` is the cached bound `file.write` (both pinned by the caller for +/// the whole dump). When `file_slot` is `None` (the `dumps` path) nothing is /// flushed: `pending` accumulates the entire pickle and the caller takes it. struct Framer { current_frame: Option>, pending: Vec, file_slot: Option, + write_slot: Option, } impl Framer { - fn new(file_slot: Option) -> Self { + fn new(file_slot: Option, write_slot: Option) -> Self { Framer { current_frame: None, pending: Vec::new(), file_slot, + write_slot, } } @@ -276,18 +286,24 @@ impl Framer { if let Some(slot) = self.file_slot { if !self.pending.is_empty() { let w_bytes = pyre_object::w_bytes_from_bytes(&self.pending); - // `call_meth` resolves `write` via getattr, which allocates a - // bound method and can relocate the freshly-built `w_bytes`; - // pin it and pass the re-read root (`w_file` is pinned at `slot`). + // Pin the freshly-built bytes and re-read both it and the + // cached callable from their roots before arbitrary Python. let _roots = pyre_object::gc_roots::push_roots(); pyre_object::gc_roots::pin_root(w_bytes); let bytes_slot = pyre_object::gc_roots::shadow_stack_len() - 1; - let w_file = pyre_object::gc_roots::shadow_stack_get(slot); - call_meth( - w_file, - "write", - &[pyre_object::gc_roots::shadow_stack_get(bytes_slot)], - )?; + if let Some(write_slot) = self.write_slot { + call_fn( + pyre_object::gc_roots::shadow_stack_get(write_slot), + &[pyre_object::gc_roots::shadow_stack_get(bytes_slot)], + )?; + } else { + let w_file = pyre_object::gc_roots::shadow_stack_get(slot); + call_meth( + w_file, + "write", + &[pyre_object::gc_roots::shadow_stack_get(bytes_slot)], + )?; + } self.pending.clear(); } } @@ -339,17 +355,24 @@ impl Framer { Some(slot) => { self.flush()?; let w_payload = pyre_object::w_bytes_from_bytes(&owned); - // Pin the freshly-built payload across `call_meth`'s `write` - // getattr (which can allocate and relocate it). + // Pin the freshly-built payload and re-read the cached + // callable from its outer root before arbitrary Python. let _roots = pyre_object::gc_roots::push_roots(); pyre_object::gc_roots::pin_root(w_payload); let payload_slot = pyre_object::gc_roots::shadow_stack_len() - 1; - let w_file = pyre_object::gc_roots::shadow_stack_get(slot); - call_meth( - w_file, - "write", - &[pyre_object::gc_roots::shadow_stack_get(payload_slot)], - )?; + if let Some(write_slot) = self.write_slot { + call_fn( + pyre_object::gc_roots::shadow_stack_get(write_slot), + &[pyre_object::gc_roots::shadow_stack_get(payload_slot)], + )?; + } else { + let w_file = pyre_object::gc_roots::shadow_stack_get(slot); + call_meth( + w_file, + "write", + &[pyre_object::gc_roots::shadow_stack_get(payload_slot)], + )?; + } } None => self.pending.extend_from_slice(&owned), } @@ -375,6 +398,7 @@ impl W_Pickler { w_class: std::ptr::null_mut(), }, w_file: pyre_object::w_none(), + w_write: pyre_object::w_none(), proto: 0, bin: false, framing: false, @@ -416,15 +440,19 @@ impl W_Pickler { // `fix_imports` gates the `_compat_pickle` py3→py2 name remap that the // protocol-< 3 save path would otherwise always apply. let proto = normalize_protocol(pyre_object::gc_roots::shadow_stack_get(protocol_slot))?; - // `file must have a 'write' attribute` (interp_pickle.py:557). - if crate::baseobjspace::findattr_result( + // `file must have a 'write' attribute` (interp_pickle.py:557). This + // check precedes the `buffer_callback` one below; `descr__new__` + // (`interp_pickle.py:1822`) orders them the other way. Measured on + // 3.14.5, a call carrying both faults reports this TypeError. + let Some(w_write) = crate::baseobjspace::findattr_result( pyre_object::gc_roots::shadow_stack_get(file_slot), "write", )? - .is_none() - { + else { return Err(PyError::type_error("file must have a 'write' attribute")); - } + }; + pyre_object::gc_roots::pin_root(w_write); + let write_slot = pyre_object::gc_roots::shadow_stack_len() - 1; if !unsafe { pyre_object::is_none(pyre_object::gc_roots::shadow_stack_get( buffer_callback_slot, @@ -439,6 +467,7 @@ impl W_Pickler { let memo = pyre_object::listobject::w_list_new(Vec::new()); let current = cur_pickler(self_slot); current.w_file = pyre_object::gc_roots::shadow_stack_get(file_slot); + current.w_write = pyre_object::gc_roots::shadow_stack_get(write_slot); current.proto = proto; current.bin = proto >= 1; current.framing = proto >= 4; @@ -485,6 +514,7 @@ impl W_Pickler { fast, w_dispatch_table, w_file, + w_write, buffer_callback, w_memo, ) = { @@ -503,6 +533,7 @@ impl W_Pickler { current.fast, current.w_dispatch_table, current.w_file, + current.w_write, current.buffer_callback, current.w_memo, ) @@ -525,6 +556,8 @@ impl W_Pickler { let obj_slot = pyre_object::gc_roots::shadow_stack_len() - 1; pyre_object::gc_roots::pin_root(w_file); let file_slot = pyre_object::gc_roots::shadow_stack_len() - 1; + pyre_object::gc_roots::pin_root(w_write); + let write_slot = pyre_object::gc_roots::shadow_stack_len() - 1; pyre_object::gc_roots::pin_root(w_memo); let memo_slot = pyre_object::gc_roots::shadow_stack_len() - 1; pyre_object::gc_roots::pin_root(buffer_callback); @@ -580,13 +613,15 @@ impl W_Pickler { let w_obj = pyre_object::gc_roots::shadow_stack_get(obj_slot); let w_memo = pyre_object::gc_roots::shadow_stack_get(memo_slot); let w_file = pyre_object::gc_roots::shadow_stack_get(file_slot); + let w_write = pyre_object::gc_roots::shadow_stack_get(write_slot); let buffer_callback = pyre_object::gc_roots::shadow_stack_get(cb_slot); let pers_func = pyre_object::gc_roots::shadow_stack_get(pers_slot); let reducer_override = pyre_object::gc_roots::shadow_stack_get(reducer_slot); let dispatch_table = pyre_object::gc_roots::shadow_stack_get(dt_slot); - pickle_core( + pickle_core_impl( w_obj, w_file, + w_write, proto, bin, framing, @@ -944,6 +979,39 @@ pub(crate) fn pickle_core( fast: bool, dispatch_table: PyObjectRef, reducer_override: PyObjectRef, +) -> Result { + pickle_core_impl( + w_obj, + w_file, + pyre_object::PY_NULL, + proto, + bin, + framing, + fix_imports, + pers_func, + buffer_callback, + w_memo, + fast, + dispatch_table, + reducer_override, + ) +} + +#[allow(clippy::too_many_arguments)] +fn pickle_core_impl( + w_obj: PyObjectRef, + w_file: PyObjectRef, + w_write: PyObjectRef, + proto: i64, + bin: bool, + framing: bool, + fix_imports: bool, + pers_func: PyObjectRef, + buffer_callback: PyObjectRef, + w_memo: PyObjectRef, + fast: bool, + dispatch_table: PyObjectRef, + reducer_override: PyObjectRef, ) -> Result { let _roots = pyre_object::gc_roots::push_roots(); pyre_object::gc_roots::pin_root(w_obj); @@ -1000,7 +1068,13 @@ pub(crate) fn pickle_core( pyre_object::gc_roots::pin_root(w_file); Some(pyre_object::gc_roots::shadow_stack_len() - 1) }; - let mut fr = Framer::new(file_slot); + let write_slot = if w_write.is_null() { + None + } else { + pyre_object::gc_roots::pin_root(w_write); + Some(pyre_object::gc_roots::shadow_stack_len() - 1) + }; + let mut fr = Framer::new(file_slot, write_slot); if proto >= 2 { fr.push(op::PROTO); fr.push(proto as u8); diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 371ac0bab5b..35e39b882e7 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -1086,11 +1086,13 @@ unsafe fn memoryview_object_destructor(obj_addr: usize) { /// Forwarded (mirrors `walk_pyframe_roots` eval.rs:496-556): /// - `f_backref` — the parent frame pointer, or the `JitVirtualRef` standing /// in for it once the JIT virtualizes an inlined callee. The vref is a GC -/// object registered with `forced` as its one traced slot, so forwarding +/// object whose `forced` and `virtual_token` slots are traced, so forwarding /// this slot greys the vref and the collector reaches the parent frame /// through it; no hop is needed here. /// - `pycode` — visited to match the walker; inert while code objects /// are Box-immortal (`is_nursery_object_start` short-circuits). +/// - `vable_token` — null, the prebuilt tracing sentinel, or the active +/// JITFRAME GCREF (`rvirtualizable.py:29`). /// - `locals_cells_stack_w` — the array pointer. A GC-managed nursery /// block forwards through its field slot and its type-9 walker owns the /// items. An old-gen GC block also visits the field slot and walks its @@ -1111,6 +1113,7 @@ unsafe fn pyframe_object_custom_trace(obj_addr: usize, f: &mut dyn FnMut(*mut ma f(&mut frame.ob_header.w_class as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); f(&mut frame.f_backref as *mut *mut PyFrame as *mut majit_ir::GcRef); f(&mut frame.pycode as *mut *const () as *mut majit_ir::GcRef); + f(&mut frame.vable_token as *mut usize as *mut majit_ir::GcRef); // locals_cells_stack_w: visit the field slot for every GC array so major // marking reaches it. A nursery array is subsequently scanned by its own @@ -1387,41 +1390,20 @@ fn build_gc() -> Box { // jitframe.py:49 — rgc.register_custom_trace_hook(JITFRAME, jitframe_trace) let jitframe_tid = gc.register_type(majit_backend::jitframe::jitframe_type_info()); debug_assert_eq!(jitframe_tid, JITFRAME_GC_TYPE_ID); - // pyre allocates jitframes via `libc::calloc` (not nursery/oldgen), - // so the collector's standard `walk_jf_roots` visitor can't - // route them through `trace_and_update_object`. Register a - // host-side tracer that invokes `jitframe_trace` directly so - // Refs pinned to frame slots are visible to GC across minor - // collections triggered by CallMallocNursery slow paths. + // Dynasm allocates jitframes off-GC, so its shadow-stack roots still need + // the host-side tracer. Cranelift's nursery JITFRAMEs use the registered + // custom trace directly. Off-GC addresses are inert when encountered in a + // traced GCREF slot, preserving the dynasm representation. majit_gc::shadow_stack::register_libc_jitframe_tracer(pyre_libc_jitframe_tracer); // virtualref.py — JIT_VIRTUAL_REF as a proper GC type. // Layout: three pointer-sized words — super_.typeptr | virtual_token | - // forced — so the size and the traced offset below are both derived from - // the struct rather than spelled out for one word width. - // - // Note (GC trace divergence). Upstream - // `virtualref.py:17-20` declares both `virtual_token` and - // `forced` as GC slots (`llmemory.GCREF` / `OBJECTPTR`); pyre - // registers only `forced` in `gc_ptr_offsets`. - // The `virtual_token` slot is intentionally outside the GC's - // view because every runtime value it can hold lives outside - // any GC heap: TOKEN_NONE (null), `token_tracing_rescall()` - // (program-lifetime leaked `Box` dummy lazily - // allocated by `allocate_tracing_rescall_dummy` and cached in - // `TRACING_RESCALL_DUMMY_PTR`), and active JITFRAME addresses - // (libc::calloc'd, see `register_libc_jitframe_tracer` above). - // The optimizer-side descriptor at - // `majit-metainterp/src/optimizeopt/virtualize.rs:make_vref_field_descr` - // still uses `Type::Ref` so `setfield_gc_r` / `getfield_gc_r` - // emit correctly; only the collector's view of the slot - // diverges. Convergence requires both `_dummy` and JITFRAME - // allocation to move under the GC. + // forced. `virtualref.py:17-20` declares both payload fields as GC slots. let vref_tid = gc.register_type(majit_gc::trace::TypeInfo::with_gc_ptrs( std::mem::size_of::(), - vec![std::mem::offset_of!( - majit_metainterp::virtualref::JitVirtualRef, - forced - )], + vec![ + std::mem::offset_of!(majit_metainterp::virtualref::JitVirtualRef, virtual_token), + std::mem::offset_of!(majit_metainterp::virtualref::JitVirtualRef, forced), + ], )); debug_assert_eq!(vref_tid, VREF_GC_TYPE_ID); // Tell the virtualref optimizer about the registered type id. @@ -3647,6 +3629,14 @@ fn build_gc() -> Box { pyre_object::gc_storage::storage_box_destructor::, pyre_object::typeobject::set_name_storage_gc_type_id, ); + // `virtualizable.py:326-330 _DUMMY`: a registered GC leaf whose address is + // the TOKEN_TRACING_RESCALL sentinel. Append it so established ids do not + // move, then publish the id before any tracing protocol can request it. + let tracing_rescall_dummy_tid = gc.register_type(TypeInfo::with_gc_ptrs( + std::mem::size_of::(), + vec![], + )); + majit_metainterp::virtualref::set_tracing_rescall_dummy_gc_type_id(tracing_rescall_dummy_tid); // rclass.py:340-346 — assign subclassrange_{min,max} to each // vtable entry. freeze_types() runs assign_inheritance_ids // (normalizecalls.py:373-389), then we write the computed ranges diff --git a/pyre/pyre-jit/tests/gc_stress.rs b/pyre/pyre-jit/tests/gc_stress.rs index 6532bc5d555..a72281b9580 100644 --- a/pyre/pyre-jit/tests/gc_stress.rs +++ b/pyre/pyre-jit/tests/gc_stress.rs @@ -1640,3 +1640,45 @@ while i < 40: "module dict move_to_end reentrant scan callback GC rooting program failed", ); } + +/// `virtualizable.py:326-330` makes `TOKEN_TRACING_RESCALL` the address of a +/// prebuilt GC object, so the sentinel has to belong to the heap that is +/// current when a traced slot holds it. `reset_gc_fresh_for_test` builds a +/// second heap and leaks the first, which leaves any sentinel minted in the +/// first outside the live heap — `is_managed_heap_object` stops recognising it, +/// and the traced `virtual_token` / `vable_token` slots would be back to +/// holding an address the collector does not own. +/// +/// Mint the sentinel between two resets and require a fresh address, which is +/// what tells the two heaps apart. This is the ordering the GC-stress harness +/// itself produces the moment any of its programs reaches a traced residual +/// call; none does today, so nothing else in this binary covers it. +#[test] +fn tracing_sentinel_is_reminted_for_a_rebuilt_heap() { + let _serial = GC_STRESS_SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let handle = std::thread::Builder::new() + .stack_size(256 * 1024 * 1024) + .spawn(|| { + init_jit_hooks(); + reset_gc_fresh_for_test(); + let first = majit_metainterp::virtualref::token_tracing_rescall(); + assert!(!first.is_null(), "sentinel must never be TOKEN_NONE"); + + reset_gc_fresh_for_test(); + let second = majit_metainterp::virtualref::token_tracing_rescall(); + assert!(!second.is_null(), "sentinel must never be TOKEN_NONE"); + assert_ne!( + first, second, + "the sentinel stayed in the heap that was replaced", + ); + + // Stable within one heap: the token protocol compares by address. + assert_eq!( + second, + majit_metainterp::virtualref::token_tracing_rescall(), + "the sentinel moved without the heap being rebuilt", + ); + }) + .expect("spawn worker thread"); + handle.join().expect("worker thread panicked"); +}