diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index a0a81a9ea6d..fb8ede4fd36 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -390,6 +390,7 @@ fn register_active_hooks(supports_guard_gc_type: bool) { Some(gc_remove_root_via_active_runtime), ); majit_gc::set_active_gc_owns_object(Some(gc_owns_object_via_active_runtime)); + majit_gc::set_active_gc_is_nursery_object(Some(gc_is_nursery_object_via_active_runtime)); majit_gc::set_active_gc_id_or_identityhash(Some(id_or_identityhash_via_active_runtime)); majit_gc::set_active_write_barrier(Some(gc_write_barrier_via_active_runtime)); majit_gc::set_active_finalizer_hooks( @@ -1672,6 +1673,19 @@ fn gc_owns_object_via_active_runtime(addr: usize) -> bool { } } +fn gc_is_nursery_object_via_active_runtime(addr: usize) -> bool { + CRANELIFT_ACTIVE_GC.with(|cell| match cell.try_borrow_mut() { + Ok(mut guard) => guard + .as_deref_mut() + .map(|gc| gc.is_nursery_object(addr)) + .unwrap_or(false), + Err(_) => CRANELIFT_ACTIVE_GC_RAW.with(|raw| match raw.get() { + Some(ptr) => unsafe { (&*ptr).is_nursery_object(addr) }, + None => false, + }), + }) +} + /// Returns true when the active GC was present and roots were /// registered; false when no GC is active and registration was a /// no-op. Callers pair the bool with `unregister_gc_roots` on Drop. diff --git a/majit/majit-backend-dynasm/src/runner.rs b/majit/majit-backend-dynasm/src/runner.rs index 8de1694b5e3..721099d708d 100644 --- a/majit/majit-backend-dynasm/src/runner.rs +++ b/majit/majit-backend-dynasm/src/runner.rs @@ -198,6 +198,7 @@ fn register_active_hooks(supports_guard_gc_type: bool) { majit_gc::set_active_heap_stats(Some(dynasm_heap_stats)); majit_gc::set_active_root_hooks(Some(dynasm_gc_add_root), Some(dynasm_gc_remove_root)); majit_gc::set_active_gc_owns_object(Some(dynasm_gc_owns_object)); + majit_gc::set_active_gc_is_nursery_object(Some(dynasm_gc_is_nursery_object)); majit_gc::set_active_gc_id_or_identityhash(Some(dynasm_id_or_identityhash)); majit_gc::set_active_write_barrier(Some(dynasm_gc_write_barrier)); majit_gc::set_active_finalizer_hooks( @@ -616,6 +617,19 @@ fn dynasm_gc_owns_object(addr: usize) -> bool { } } +fn dynasm_gc_is_nursery_object(addr: usize) -> bool { + DYNASM_ACTIVE_GC.with(|cell| match cell.try_borrow() { + Ok(guard) => guard + .as_deref() + .map(|gc| gc.is_nursery_object(addr)) + .unwrap_or(false), + Err(_) => DYNASM_ACTIVE_GC_RAW.with(|raw| match raw.get() { + Some(ptr) => unsafe { (&*ptr).is_nursery_object(addr) }, + None => false, + }), + }) +} + /// `gc.py:51` malloc-helper OOM signaling. /// /// `do_malloc_fixedsize_clear` raises `MemoryError` on failure; diff --git a/majit/majit-backend-wasm/src/codegen.rs b/majit/majit-backend-wasm/src/codegen.rs index dea0058d281..5d0bd9a5a82 100644 --- a/majit/majit-backend-wasm/src/codegen.rs +++ b/majit/majit-backend-wasm/src/codegen.rs @@ -33,14 +33,8 @@ const SLOT_SIZE: u64 = 8; /// (al, ah, bl, bh, mid1). const UMULHI_SCRATCH: u32 = 5; -/// Call area layout (fixed offsets from frame_ptr). +/// Call area layout in the historical fixed frame geometry. const CALL_RESULT_OFS: u64 = 2000; -/// First frame *slot* index occupied by the fixed call area. Frame value slots -/// (inputs at entry, fail-arg spills at guard exit) occupy `[1, 1 + max(num -/// inputs, max fail args))`; they must stay below this index, or they clobber -/// the call area and — past `HOME_SLOT_BASE` — the Ref-home region. A trace that -/// would exceed it is declined in `build_wasm_module`. -pub const CALL_AREA_FIRST_SLOT: u64 = CALL_RESULT_OFS / SLOT_SIZE; const CALL_FUNC_OFS: u64 = 2008; const CALL_NARGS_OFS: u64 = 2016; const CALL_ARGS_OFS: u64 = 2024; @@ -48,14 +42,14 @@ const CALL_ARGS_OFS: u64 = 2024; /// Minimum frame allocation size in bytes to accommodate the call area. pub const MIN_FRAME_BYTES: usize = 2024 + 16 * 8; // 16 max call args -/// Per-token layout of a wasm execution frame. Host entry frames retain the -/// historical [`MIN_FRAME_BYTES`] allocation floor, but generated code and CA -/// nursery frames use these offsets. This mirrors `jitframe.py`'s -/// `JITFRAME_FIXED_SIZE + frame_depth`: a frame carries the slots its token -/// actually needs, rather than a backend-wide slot floor. +/// Per-token layout of a wasm execution frame. Every frozen geometry carries +/// the host-trampoline call area, so a later chained bridge can use it without +/// changing its source token's frame offsets. CA callee frames alone allocate +/// the prefix ending after the Ref homes; the tail is protected by the +/// trampoline-decline floor in `compile_bridge`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct FrameGeometry { - /// Number of data slots before the call trampoline (including frame[0]). + /// Number of value slots before the dispatch key (including frame[0]). pub value_slots: usize, /// Byte offset of the call trampoline result word. pub call_result_ofs: u64, @@ -68,7 +62,11 @@ pub struct FrameGeometry { pub home_slot_base: u64, /// Number of Ref-home slots the layout reserves. pub home_slots: usize, - /// Bytes in the JitFrame item area required by this layout. + /// Bytes through the end of Ref homes. CA callee frames allocate exactly + /// this many item bytes; the tail call area is intentionally omitted. + pub ca_frame_bytes: u32, + /// Full bytes in the frame layout, including the tail call area. Host entry + /// frames and every chained bridge use this geometry and allocation size. pub frame_bytes: u32, } @@ -87,22 +85,26 @@ impl FrameGeometry { dispatch_key_ofs: DISPATCH_KEY_OFS, home_slot_base: HOME_SLOT_BASE, home_slots: 0, + ca_frame_bytes: HOME_SLOT_BASE as u32, frame_bytes: (MIN_FRAME_BYTES + SLOT_SIZE as usize) as u32, } } - /// Compact geometry for one token. `value_slots` includes frame[0], so - /// the call area begins immediately after the greatest positional input or - /// fail-arg slot used by that token. + /// Compact frozen geometry for one token: + /// `[value slots | dispatch key | Ref homes | call area]`. + /// `value_slots` includes frame[0]. The trailing call area is always + /// present, even for direct-only source traces, because later bridges are + /// compiled against this immutable geometry. pub fn compact(value_slots: usize, home_slots: usize) -> Self { let value_slots = value_slots.max(1); - let call_result_ofs = (value_slots as u64) * SLOT_SIZE; + let dispatch_key_ofs = (value_slots as u64) * SLOT_SIZE; + let home_slot_base = dispatch_key_ofs + SLOT_SIZE; + let ca_frame_bytes = home_slot_base + home_slots as u64 * SLOT_SIZE; + let call_result_ofs = ca_frame_bytes; let call_func_ofs = call_result_ofs + SLOT_SIZE; let call_nargs_ofs = call_func_ofs + SLOT_SIZE; let call_args_ofs = call_nargs_ofs + SLOT_SIZE; - let dispatch_key_ofs = call_result_ofs + Self::CALL_AREA_SLOTS as u64 * SLOT_SIZE; - let home_slot_base = dispatch_key_ofs + SLOT_SIZE; - let frame_bytes = home_slot_base + home_slots as u64 * SLOT_SIZE; + let frame_bytes = call_result_ofs + Self::CALL_AREA_SLOTS as u64 * SLOT_SIZE; Self { value_slots, call_result_ofs, @@ -112,6 +114,7 @@ impl FrameGeometry { dispatch_key_ofs, home_slot_base, home_slots, + ca_frame_bytes: ca_frame_bytes as u32, frame_bytes: frame_bytes as u32, } } @@ -125,18 +128,17 @@ impl FrameGeometry { /// then the trace reloads the live Ref locals from their homes — making object /// movement transparent without rooting Refs that never cross a collection. /// -/// Placed past the call area so it never overlaps the fail-index / input / -/// output slots (which sit below the call area at offset 2000) or the call -/// trampoline area. Inert while `wasm_jit_alloc` is no-collect (epic B): the +/// In compact geometries this region follows the dispatch key and precedes the +/// trailing call area. Inert while `wasm_jit_alloc` is no-collect (epic B): the /// extra stores write a region nothing reads until the allocator collects. pub const HOME_SLOT_BASE: u64 = MIN_FRAME_BYTES as u64 + SLOT_SIZE; -/// Resume-at-LABEL dispatch key (one reserved frame slot, between the call -/// area and the Ref-home region). 0 = preamble/host entry (the `vec![0i64]` +/// Historical fixed-geometry resume-at-LABEL dispatch key (one reserved frame +/// slot, between the call area and the Ref-home region). 0 = preamble/host entry (the `vec![0i64]` /// frame is always 0 here on a fresh `execute_token`); non-zero = a /// loop-closing bridge re-entering a single-label peeled loop at its LABEL, -/// skipping the preamble. Distinct from every value/fail-arg slot (< 2000) and -/// the call trampoline (2000..2152); homes follow it at `HOME_SLOT_BASE`. +/// skipping the preamble. Compact geometries derive this offset from their +/// value-slot count and put the call area after the homes. pub const DISPATCH_KEY_OFS: u64 = MIN_FRAME_BYTES as u64; const _: () = assert!(HOME_SLOT_BASE == DISPATCH_KEY_OFS + SLOT_SIZE); @@ -148,6 +150,10 @@ fn mem64(offset: u64) -> MemArg { } } +fn mem32(offset: u64) -> MemArg { + memarg(offset, 2) +} + fn memarg(offset: u64, align: u32) -> MemArg { MemArg { offset, @@ -675,8 +681,15 @@ fn emit_reload_frame_if_necessary( sink: &mut InstructionSink<'_>, residual_type_base: Option, ca_reload_fn_ptr: i64, + jf_top_addr: Option, ) { - if let Some(base) = residual_type_base { + if let Some(top_addr) = jf_top_addr { + // assembler.py:1369-1377: reload the possibly-forwarded top JitFrame + // directly from the shadow-stack cell. Unlike the helper-table call, + // this does not need the residual direct-call type to be declared. + emit_ca_reload_top(sink, top_addr); + sink.local_set(0); + } else if let Some(base) = residual_type_base { sink.i32_const(ca_reload_fn_ptr as i32); sink.call_indirect(0, base); sink.i32_wrap_i64(); @@ -686,6 +699,47 @@ fn emit_reload_frame_if_necessary( } } +/// CA-arm-only variant of [`emit_reload_frame_if_necessary`]. The direct CA +/// configuration owns an inline shadow-stack top cell; all other call sites +/// retain their pre-existing helper reload. +fn emit_reload_ca_frame_if_necessary( + sink: &mut InstructionSink<'_>, + residual_type_base: Option, + ca_reload_fn_ptr: i64, + ca_inline: Option, +) { + if let Some(inline) = ca_inline { + debug_assert!(residual_type_base.is_some()); + emit_ca_reload_top(sink, inline.jf_top_addr); + sink.local_set(0); + } else { + emit_reload_frame_if_necessary(sink, residual_type_base, ca_reload_fn_ptr, None); + } +} + +/// assembler.py `_reload_frame_if_necessary`: `top[-WORD]` is the top +/// jitframe pointer. The wasm CA ABI carries its ITEMS base in local 0. +fn emit_ca_reload_top(sink: &mut InstructionSink<'_>, top_addr: u32) { + sink.i32_const(top_addr as i32); + sink.i32_load(mem32(0)); + sink.i32_const(4); + sink.i32_sub(); + sink.i32_load(mem32(0)); + sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); + sink.i32_add(); +} + +/// While a CA callee is pushed, its caller's `jf_ptr` is `top[-3 * WORD]`. +fn emit_ca_reload_caller(sink: &mut InstructionSink<'_>, top_addr: u32) { + sink.i32_const(top_addr as i32); + sink.i32_load(mem32(0)); + sink.i32_const(12); + sink.i32_sub(); + sink.i32_load(mem32(0)); + sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); + sink.i32_add(); +} + /// Reload the Ref operands which the CA arm resolves only after its collecting /// callee-frame allocation. Unlike ordinary post-call reloads, these are live /// *at* the CALL_ASSEMBLER op, not after it. @@ -896,6 +950,40 @@ fn has_call_ops(ops: &[Op]) -> bool { }) } +/// Whether this trace emits a host `jit_call` / `jit_call_compact` trampoline +/// invocation. CA frames are movable nursery objects, while the host +/// trampoline writes its result back through the pre-call frame pointer, so +/// `compile_bridge` uses this exact lowering census to keep such traces off a +/// live CA frame. +/// +/// Keep this in lockstep with the individual emission arms below: the uniform +/// i64 residual family, `New*`, and write barriers are direct under +/// `WASM_DIRECT_RESIDUAL_CALL`; non-uniform CALLs and string allocation retain +/// the trampoline. When the direct family is disabled, all of the existing +/// call-area users return to the trampoline baseline. +pub fn has_trampoline_calls(inputargs: &[InputArg], ops: &[Op], emit_ca: bool) -> bool { + let ref_values = RefValues::collect(inputargs, ops); + if !WASM_DIRECT_RESIDUAL_CALL { + return has_call_ops(ops) || has_ref_store_op(ops, &ref_values); + } + + ops.iter().any(|op| match op.opcode { + // `build_function` handles an enabled CA `CallAssemblerR` before the + // generic CALL arm, lowering it directly to the source-loop table + // slot. It therefore never uses the host call area. + OpCode::CallAssemblerR if emit_ca => false, + // These arms have no direct helper lowering. + OpCode::Newstr | OpCode::Newunicode => true, + // Every residual CALL uses the trampoline unless its exact lowering + // predicate supplies an i64 helper ABI. + _ if op.opcode.is_call() => direct_helper_i64_arity(op, &ref_values).is_none(), + // `New*` and ref-store write barriers are covered by + // `direct_helper_i64_arity`, so their direct-family arms do not touch + // the frame call area. + _ => false, + }) +} + fn collect_guards_and_vars(inputargs: &[InputArg], ops: &[Op]) -> (Vec, u32) { let mut guards = Vec::new(); let mut max_var: u32 = 0; @@ -988,10 +1076,13 @@ pub struct CaParams { /// into the source loop). Only set by `compile_bridge` for a self-recursive /// single-int bridge; `compile_loop` never sets it. pub emit_ca: bool, - /// Bytes to reserve per callee frame (the GC `JitFrame`'s data region, i.e. - /// its Signed item area). Sized for the SOURCE loop, widened to also fit THIS - /// bridge (which reuses the frame when the loop's guard-exit chains back into - /// it). The alloc trampoline derives the JitFrame item count from it. + /// Bytes to reserve per CA callee frame (the GC `JitFrame`'s data region, + /// i.e. its Signed item area). This is the source geometry's prefix through + /// the Ref homes, excluding its tail call area. The trampoline-decline + /// floor in `WasmBackend::compile_bridge` (`source_ca_active && + /// bridge_has_trampoline_calls`) guarantees that no trampoline-lowered op + /// runs on this movable frame, so the omitted tail is unreachable. The alloc + /// trampoline derives the JitFrame item count from this exact byte count. pub callee_frame_bytes: u32, /// `fail_index` the SOURCE loop's DoneWithThisFrame Finish writes to frame[0] /// on the base-case return. The CA arm treats this — or this bridge's own @@ -1022,6 +1113,11 @@ pub struct CaParams { /// called after the recursive call to recover this level's possibly-moved /// nursery frame from the jitframe shadow stack. pub ca_reload_fn_ptr: i64, + /// Address of the active jitframe shadow-stack top cell, baked for every + /// trace body so post-collecting-call local-0 reloads can match + /// assembler.py without a helper round trip. `None` keeps the existing + /// helper/trampoline behavior when compilation has no active GC. + pub jf_top_addr: Option, /// `__indirect_function_table` slot of /// `lib.rs::wasm_jit_ca_reload_caller_frame`, called while the callee is /// still pushed to recover this invocation's possibly-moved local-0 frame. @@ -1030,6 +1126,19 @@ pub struct CaParams { /// callee frame's CA input + home Ref slots; baked into each frame's /// `jf_gcmap` field at alloc time. pub callee_gcmap_ptr: i64, + /// Active-GC state for the direct CA-only inline allocation/frame path. + /// `None` retains the helpers (including under gc_stress). + pub inline: Option, +} + +/// Direct CA fast-path values baked at bridge compilation time. +#[derive(Clone, Copy)] +pub struct CaInlineParams { + pub nursery_free_addr: u32, + pub nursery_top_addr: u32, + pub jf_top_addr: u32, + pub jf_limit_addr: u32, + pub jitframe_tid: u32, } /// Inline nursery-bump fast-path parameters for `New`/`NewWithVtable` @@ -1121,11 +1230,10 @@ pub fn build_wasm_module( let bridge_dispatch = cells_base != 0; // Frame value slots (inputs at entry, fail-arg spills at guard exit) occupy - // `[1, 1 + max(num inputs, max fail args))`. They sit below the fixed call - // area and the Ref-home region, which are at constant offsets; a trace whose - // value slots would reach the call area must be declined, or those stores - // silently clobber the call trampoline / home roots. (Pre-existing for the - // call area; the home region inherits the same bound.) + // `[1, 1 + max(num inputs, max fail args))`. They precede the dispatch key, + // Ref homes, and the always-present tail call area; a chained bridge must + // fit the source token's frozen value-slot count before it can share that + // frame. let max_fail_args = guards .iter() .map(|g| g.fail_arg_refs.len()) @@ -1166,17 +1274,11 @@ pub fn build_wasm_module( // geometry. `compile_bridge` rejects a bridge that needs more slots, so // no global floor or speculative slack is needed here. - // A ref-storing store needs the `jit_call` import for its write barrier, - // even when the trace has no `New*`/CALL of its own. The CA arm needs it too - // for the callee-frame GC-alloc / shadow-stack-pop trampolines (an emit_ca - // bridge always materializes its callee frame via NewWithVtable, so this is - // already true in practice — stated explicitly for robustness). - let needs_call = has_call_ops(ops) || has_ref_store_op(ops, &ref_values) || ca.emit_ca; - // The shared indirect-function table is imported for `jit_call`'s residual - // dispatch, the epilogue's bridge `call_indirect`, and the CA arm's - // self-recursive `call_indirect`. - let needs_table = needs_call || bridge_dispatch || ca.emit_ca; - + // This exact lowering census controls the host-trampoline import. Direct + // residual helpers, including the CA arm's inline fast path, use + // `call_indirect` and need no import, although their frozen frame still + // keeps the tail call area for future bridges. + let needs_call = has_trampoline_calls(inputargs, ops, ca.emit_ca); // In-module residual calls (`WASM_DIRECT_RESIDUAL_CALL`): the largest // eligible `(i64×n)->i64` arity in this trace — residual CALLs (word // result or word-ABI void) plus the `New*` / write-barrier helper @@ -1205,6 +1307,12 @@ pub fn build_wasm_module( } else { None }; + // The shared indirect-function table backs direct residual helpers as well + // as host-trampoline dispatch, chained bridges, and CA recursion. + let needs_table = needs_call || bridge_dispatch || residual_max_arity.is_some() || ca.emit_ca; + // `ca.emit_ca` forces the direct helper family to include arities 0..=2, + // so all CA frame-helper trampoline `else` arms below are baseline-only. + debug_assert!(!ca.emit_ca || residual_max_arity.is_some()); let mut module = Module::new(); @@ -1385,6 +1493,10 @@ fn build_function( // `ca.deopt_helper_slot` for a deopted callee. ca_helper_type_idx: u32, ) -> Result { + // The CA arm requires residual types (the setup above forces arity >= 2 + // while `WASM_DIRECT_RESIDUAL_CALL` is enabled). Its `jit_call` fallback + // branches are retained solely for the direct-family-disabled baseline. + debug_assert!(!ca.emit_ca || residual_type_base.is_some()); // Value locals occupy `1 ..= num_vars`; reserve `UMULHI_SCRATCH` extra i64 // locals past them (`num_vars+1 ..= num_vars+UMULHI_SCRATCH`) as scratch for // the `UintMulHigh` 32-bit-split expansion (`emit_umulhi`). One i32 local @@ -1407,7 +1519,12 @@ fn build_function( let mut func = Function::new(vec![ (num_vars + UMULHI_SCRATCH, ValType::I64), ( - base_i32_locals + if nursery.is_some() { 2 } else { 0 }, + base_i32_locals + + if nursery.is_some() || ca.inline.is_some() { + 2 + } else { + 0 + }, ValType::I32, ), ]); @@ -2632,7 +2749,103 @@ fn build_function( // `ca_cfp_local = frame_base + FIRST_ITEM_OFFSET` is the // bespoke-layout frame pointer — every `mem64(OFS)` below is // relative to it, exactly as the source loop reads its local 0. - if let Some(base) = residual_type_base { + let ca_depth = ca.callee_frame_bytes as usize / std::mem::size_of::(); + let ca_payload_size = majit_backend::jitframe::JitFrame::alloc_size(ca_depth); + let ca_total_size = + ((GcHeader::SIZE + ca_payload_size).max(GcHeader::MIN_NURSERY_OBJ_SIZE) + 7) + & !7; + if let (Some(base), Some(inline)) = (residual_type_base, ca.inline) { + // rewrite.py's nursery fast path plus assembler.py's inline + // shadow-stack header. The `memory.fill` is deliberate: + // home slots are read as roots before every definition, and + // guard/deopt fail slots may be read by the host. Do not rely + // on nursery reset's pre-existing memset for either class. + sink.i32_const(inline.nursery_free_addr as i32); + sink.i32_load(mem32(0)); + sink.local_tee(alloc_scratch_local); + sink.i32_const(ca_total_size as i32); + sink.i32_add(); + sink.i32_const(inline.nursery_top_addr as i32); + sink.i32_load(mem32(0)); + sink.i32_gt_u(); + sink.i32_const(inline.jf_top_addr as i32); + sink.i32_load(mem32(0)); + sink.local_tee(alloc_size_local); + sink.i32_const(8); + sink.i32_add(); + sink.i32_const(inline.jf_limit_addr as i32); + sink.i32_load(mem32(0)); + sink.i32_gt_u(); + sink.i32_or(); + sink.if_(BlockType::Result(ValType::I64)); + // Slow path collects/allocates and performs init + push. + sink.i64_const(ca.callee_frame_bytes as i64); + sink.i64_const(ca.callee_gcmap_ptr); + sink.i32_const(ca.ca_alloc_fn_ptr as i32); + sink.call_indirect(0, base + 2); + sink.else_(); + // Commit the nursery bump, then write the exact young + // `GcHeader::new(jitframe_tid)` word (flags are clear). + sink.i32_const(inline.nursery_free_addr as i32); + sink.local_get(alloc_scratch_local); + sink.i32_const(ca_total_size as i32); + sink.i32_add(); + sink.i32_store(mem32(0)); + sink.local_get(alloc_scratch_local); + sink.i64_const(inline.jitframe_tid as i64); + sink.i64_store(mem64(0)); + // Explicitly initialise the complete JitFrame payload and + // item area, then replicate JitFrame::init + jf_gcmap. + sink.local_get(alloc_scratch_local); + sink.i32_const(GcHeader::SIZE as i32); + sink.i32_add(); + sink.i32_const(0); + sink.i32_const(ca_payload_size as i32); + sink.memory_fill(0); + for offset in [ + majit_backend::jitframe::JF_FRAME_INFO_OFS, + majit_backend::jitframe::JF_DESCR_OFS, + majit_backend::jitframe::JF_FORCE_DESCR_OFS, + majit_backend::jitframe::JF_SAVEDATA_OFS, + majit_backend::jitframe::JF_GUARD_EXC_OFS, + majit_backend::jitframe::JF_FORWARD_OFS, + ] { + sink.local_get(alloc_scratch_local); + sink.i32_const(0); + sink.i32_store(mem32(GcHeader::SIZE as u64 + offset as u64)); + } + sink.local_get(alloc_scratch_local); + sink.i32_const(ca.callee_gcmap_ptr as i32); + sink.i32_store(mem32( + GcHeader::SIZE as u64 + majit_backend::jitframe::JF_GCMAP_OFS as u64, + )); + sink.local_get(alloc_scratch_local); + sink.i32_const(ca_depth as i32); + sink.i32_store(mem32( + GcHeader::SIZE as u64 + majit_backend::jitframe::JF_FRAME_OFS as u64, + )); + // Push `[is_minor=1, jf_ptr]`; the limit check above made + // these stores safe, so the helper's overflow assertion is + // retained only on the slow path. + sink.local_get(alloc_size_local); + sink.i32_const(1); + sink.i32_store(mem32(0)); + sink.local_get(alloc_size_local); + sink.local_get(alloc_scratch_local); + sink.i32_const(GcHeader::SIZE as i32); + sink.i32_add(); + sink.i32_store(mem32(4)); + sink.i32_const(inline.jf_top_addr as i32); + sink.local_get(alloc_size_local); + sink.i32_const(8); + sink.i32_add(); + sink.i32_store(mem32(0)); + sink.local_get(alloc_scratch_local); + sink.i32_const(GcHeader::SIZE as i32); + sink.i32_add(); + sink.i64_extend_i32_u(); + sink.end(); + } else if let Some(base) = residual_type_base { sink.i64_const(ca.callee_frame_bytes as i64); sink.i64_const(ca.callee_gcmap_ptr); sink.i32_const(ca.ca_alloc_fn_ptr as i32); @@ -2667,9 +2880,12 @@ fn build_function( // resolving inputs through local-0-relative homes. The // trampoline path intentionally keeps the A0-era assumption: // its scratch writes themselves dereference stale local 0. - if let Some(base) = residual_type_base { + if let (Some(_base), Some(inline)) = (residual_type_base, ca.inline) { + emit_ca_reload_caller(&mut sink, inline.jf_top_addr); + sink.local_set(0); + } else if let Some(base) = residual_type_base { sink.i32_const(ca.ca_reload_caller_fn_ptr as i32); - sink.call_indirect(0, base + 0); + sink.call_indirect(0, base); sink.i32_wrap_i64(); sink.local_set(0); } @@ -2696,7 +2912,10 @@ fn build_function( // callee frame. Deeper levels have already popped, so the // jitframe shadow-stack top is this level's frame; reload its // ITEMS base before reading F'[0] or F'[1]. - if let Some(base) = residual_type_base { + if let (Some(_base), Some(inline)) = (residual_type_base, ca.inline) { + emit_ca_reload_top(&mut sink, inline.jf_top_addr); + sink.i64_extend_i32_u(); + } else if let Some(base) = residual_type_base { sink.i32_const(ca.ca_reload_fn_ptr as i32); sink.call_indirect(0, base + 0); } else { @@ -2753,9 +2972,12 @@ fn build_function( // the trampoline-only configuration retains its A0-era stale- // local-0 limitation because its scratch writes cannot reload it // safely. - if let Some(base) = residual_type_base { + if let (Some(_base), Some(inline)) = (residual_type_base, ca.inline) { + emit_ca_reload_caller(&mut sink, inline.jf_top_addr); + sink.local_set(0); + } else if let Some(base) = residual_type_base { sink.i32_const(ca.ca_reload_caller_fn_ptr as i32); - sink.call_indirect(0, base + 0); + sink.call_indirect(0, base); sink.i32_wrap_i64(); sink.local_set(0); } @@ -2769,7 +2991,14 @@ fn build_function( // LIFO) via `wasm_jit_ca_pop_frame` — same direct-vs-trampoline // split as the alloc above (the pop only shrinks the shadow // stack; it never allocates or collects). - if let Some(base) = residual_type_base { + if let (Some(_base), Some(inline)) = (residual_type_base, ca.inline) { + sink.i32_const(inline.jf_top_addr as i32); + sink.i32_const(inline.jf_top_addr as i32); + sink.i32_load(mem32(0)); + sink.i32_const(8); + sink.i32_sub(); + sink.i32_store(mem32(0)); + } else if let Some(base) = residual_type_base { sink.local_get(ca_cfp_local); sink.i64_extend_i32_u(); sink.i32_const(ca.ca_pop_fn_ptr as i32); @@ -2797,7 +3026,12 @@ fn build_function( // and its home is not written until the store-on-def below, so a // reload would clobber it with the home's pre-call (stale) value. let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); - emit_reload_frame_if_necessary(&mut sink, residual_type_base, ca.ca_reload_fn_ptr); + emit_reload_ca_frame_if_necessary( + &mut sink, + residual_type_base, + ca.ca_reload_fn_ptr, + ca.inline, + ); emit_reload_refs_from_homes(&mut sink, ref_homes, &liveness, op_idx, skip, frame); } @@ -2856,6 +3090,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, @@ -2885,6 +3120,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, ref_homes, &liveness, op_idx, None, frame, @@ -3000,6 +3236,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, @@ -3115,6 +3352,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, ref_homes, &liveness, op_idx, skip, frame, @@ -3218,6 +3456,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, @@ -3286,6 +3525,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, @@ -3349,6 +3589,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, @@ -3464,6 +3705,7 @@ fn build_function( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, + ca.jf_top_addr, ); emit_reload_refs_from_homes( &mut sink, ref_homes, &liveness, op_idx, skip, frame, @@ -4164,3 +4406,19 @@ fn emit_unary_vi( sink.local_set(1 + vi); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compact_geometry_keeps_tail_call_area_out_of_ca_prefix() { + let frame = FrameGeometry::compact(32, 16); + assert_eq!(frame.dispatch_key_ofs, 32 * SLOT_SIZE); + assert_eq!(frame.home_slot_base, 33 * SLOT_SIZE); + assert_eq!(frame.ca_frame_bytes, 392); + assert_eq!(frame.call_result_ofs, frame.ca_frame_bytes as u64); + assert_eq!(frame.call_args_ofs, 416); + assert_eq!(frame.frame_bytes, 544); + } +} diff --git a/majit/majit-backend-wasm/src/failguard.rs b/majit/majit-backend-wasm/src/failguard.rs index ce20315bcec..1e0263a267a 100644 --- a/majit/majit-backend-wasm/src/failguard.rs +++ b/majit/majit-backend-wasm/src/failguard.rs @@ -219,6 +219,11 @@ pub struct CompiledWasmLoop { /// Geometry frozen when this token was first compiled. Every bridge /// chained onto it is emitted against this exact layout. pub frame: crate::codegen::FrameGeometry, + /// True when this loop or any successfully chained bridge uses the host + /// residual-call trampoline. A CA callee frame is movable, but that + /// trampoline retains the pre-call frame pointer, so `compile_bridge` must + /// not enable the CA arm for this source token. + pub has_trampoline_calls: Cell, /// Base address (shared linear memory) of this loop's per-guard bridge-slot /// cell array — one i32 per `fail_index`, `0` = no bridge. The trace's /// epilogue reads `cells[fail_index]` and `compile_bridge` writes a bridge's @@ -286,6 +291,17 @@ pub struct CompiledWasmLoop { pub ca_active: Cell, } +impl CompiledWasmLoop { + /// Incorporate the normal (non-CA unless this bridge is the candidate) + /// codegen census for a bridge after it has been chained onto this token. + /// Every earlier bridge remains reachable from a later CA recursion's + /// guard exits, so its host trampoline use also rules out CA. + pub fn record_chained_bridge_trampoline_calls(&self, bridge_has_trampoline_calls: bool) { + self.has_trampoline_calls + .set(self.has_trampoline_calls.get() || bridge_has_trampoline_calls); + } +} + impl Drop for CompiledWasmLoop { fn drop(&mut self) { // Retract this loop's published label targets so a later bridge @@ -307,3 +323,47 @@ impl Drop for CompiledWasmLoop { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn token_with_trampoline_census(has_trampoline_calls: bool) -> CompiledWasmLoop { + CompiledWasmLoop { + trace_id: 0, + input_types: Vec::new(), + func_handle: 0, + fail_descrs: RefCell::new(Vec::new()), + num_inputs: 0, + max_output_slots: 0, + num_ref_homes: 0, + frame: crate::codegen::FrameGeometry::fixed(), + has_trampoline_calls: Cell::new(has_trampoline_calls), + bridge_cells_base: 0, + num_guard_cells: 0, + has_preamble: false, + label_descrs: Vec::new(), + guard_fail_arg_advanced: Vec::new(), + bridge_descr_ranges: RefCell::new(Vec::new()), + chained_trace_meta: RefCell::new(std::collections::HashMap::new()), + _bridge_cells_owner: None, + _bridge_owned_cells: RefCell::new(Vec::new()), + ca_active: Cell::new(false), + } + } + + #[test] + fn chained_bridge_trampoline_census_is_orred_into_token() { + let token = token_with_trampoline_census(false); + token.record_chained_bridge_trampoline_calls(false); + assert!(!token.has_trampoline_calls.get()); + + token.record_chained_bridge_trampoline_calls(true); + assert!(token.has_trampoline_calls.get()); + + // A later clean bridge cannot erase an earlier chained bridge's + // trampoline census before a CA bridge is considered. + token.record_chained_bridge_trampoline_calls(false); + assert!(token.has_trampoline_calls.get()); + } +} diff --git a/majit/majit-backend-wasm/src/lib.rs b/majit/majit-backend-wasm/src/lib.rs index 3f61c20b3b3..748708c54af 100644 --- a/majit/majit-backend-wasm/src/lib.rs +++ b/majit/majit-backend-wasm/src/lib.rs @@ -30,7 +30,8 @@ use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; /// decline (TEMP, for the resume-at-last-label measurement): 8 = JUMP descr /// did not resolve (target_ord None), 9 = target_ord Some but != last label, /// 10 = arity mismatch, 11 = loop-closing bridge advances no loop-carried value -/// (guard side-trace that would livelock the chained loop). +/// (guard side-trace that would livelock the chained loop), 15 = declined CA +/// because a trace would use the host call trampoline on a movable CA frame. pub static BRIDGE_DIAG: [AtomicU64; 16] = { const Z: AtomicU64 = AtomicU64::new(0); [Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z] @@ -53,7 +54,9 @@ fn diag_bump(i: usize) { // A source token is compiled before a later guard may become a CA bridge. // Freeze modest room for that bridge at first compilation; a later trace that // exceeds either bound is declined rather than changing the live frame's -// offsets. Fib's bridge needs about 20 positional slots and 8 homes. +// offsets. The fib CA path measured raw maxima of 9 positional slots and 15 +// Ref homes, but that is not a bridge fail-arg census for the full suite; keep +// both existing floors until such a census establishes smaller safe bounds. const FROZEN_CHAIN_VALUE_SLOTS: usize = 32; const FROZEN_CHAIN_REF_HOMES: usize = 16; @@ -378,6 +381,53 @@ fn nursery_alloc_params(ops: &[Op]) -> Option { })? } +/// Assemble the direct CA arm's fixed-size nursery/frame parameters. This is +/// deliberately separate from ordinary `New*` eligibility: a CA frame needs +/// both the nursery words and the JitFrame shadow-stack top/limit cells. +/// Missing active GC (or gc_stress) leaves the pre-existing helper path intact. +fn ca_inline_params(frame_bytes: u32) -> Option { + if majit_gc::gc_stress_enabled() { + return None; + } + let jitframe_tid = wasm_jitframe_tid(); + let depth = frame_bytes as usize / std::mem::size_of::(); + let total = ((majit_gc::header::GcHeader::SIZE + + majit_backend::jitframe::JitFrame::alloc_size(depth)) + .max(majit_gc::header::GcHeader::MIN_NURSERY_OBJ_SIZE) + + 7) + & !7; + with_wasm_active_gc(|gc| { + assert_ne!( + jitframe_tid, 0, + "wasm CA inline frame path requires the registered JitFrame type id" + ); + if total > gc.max_nursery_object_size() || !gc.type_alloc_is_plain(jitframe_tid) { + return None; + } + let nursery_free_addr = gc.nursery_free_addr(); + let nursery_top_addr = gc.nursery_top_addr(); + let jf_top_addr = majit_gc::shadow_stack::get_root_stack_top_addr(); + let jf_limit_addr = majit_gc::shadow_stack::get_root_stack_limit_addr(); + (nursery_free_addr != 0 && nursery_top_addr != 0 && jf_top_addr != 0 && jf_limit_addr != 0) + .then_some(codegen::CaInlineParams { + nursery_free_addr: nursery_free_addr as u32, + nursery_top_addr: nursery_top_addr as u32, + jf_top_addr: jf_top_addr as u32, + jf_limit_addr: jf_limit_addr as u32, + jitframe_tid, + }) + })? +} + +/// Address of the active jitframe shadow-stack top cell for ordinary trace +/// body reloads. This does not depend on nursery fast-path eligibility: the +/// reload is valid whenever a GC is active at compilation time. +fn jf_top_addr() -> Option { + with_wasm_active_gc(|_| majit_gc::shadow_stack::get_root_stack_top_addr()) + .and_then(|addr| u32::try_from(addr).ok()) + .filter(|&addr| addr != 0) +} + /// `majit_gc::CollectOldgenFn` installed by `set_gc_allocator`. Drives the /// interpreter-safepoint non-moving old-gen major (`gc_interp::safepoint`, /// default-on on wasm) through the active GC. Needs mutable access, so it @@ -530,6 +580,8 @@ pub extern "C" fn wasm_jit_write_barrier(obj: i64) -> i64 { /// frame's interior as a smaller frame's slots. pub extern "C" fn wasm_jit_ca_alloc_frame(frame_bytes: i64, gcmap_ptr: i64) -> i64 { use majit_backend::jitframe::JitFrame; + assert!(frame_bytes >= 0); + assert_eq!(frame_bytes as usize % std::mem::size_of::(), 0); let depth = frame_bytes as usize / std::mem::size_of::(); // Slice A1: collecting nursery allocation, matching rewrite.py's // `gen_malloc_nursery_varsize_frame`. The caller frame remains rooted at @@ -608,6 +660,14 @@ fn build_callee_gcmap( indices.push((frame.home_slot_base as usize + h * 8) / sign); } let max_index = indices.iter().copied().max().unwrap_or(0); + // `wasm_jit_ca_alloc_frame` sets `jf_frame` from `ca_frame_bytes`, not the + // full geometry. Inputs and homes must therefore fit that actual item + // allocation; fail/deopt outputs live in the low value slots and are + // covered by the same bound. + debug_assert!( + max_index < frame.ca_frame_bytes as usize / sign, + "CA gcmap exceeds the allocated JitFrame item area" + ); let num_words = max_index / bits_per_word + 1; let mut buf = vec![0usize; 1 + num_words]; buf[0] = num_words; @@ -1171,6 +1231,9 @@ impl majit_backend::Backend for WasmBackend { } let ops_owned: Vec = ops.iter().map(|rc| (**rc).clone()).collect(); let ops: &[Op] = &ops_owned; + // This must use the same direct-vs-trampoline predicates as codegen: + // a CA callee runs this source-loop body on a movable nursery frame. + let has_trampoline_calls = codegen::has_trampoline_calls(inputargs, ops, false); // Decline traces the wasm backend cannot compile correctly, so the // metainterp falls back to the interpreter (correct, if unaccelerated) @@ -1200,12 +1263,13 @@ impl majit_backend::Backend for WasmBackend { let alloc_array_fn_ptr = wasm_jit_alloc_array as *const () as usize as i64; let wb_fn_ptr = wasm_jit_write_barrier as *const () as usize as i64; // Freeze this token's generated frame layout at first compilation. - // `jitframe.py` sizes native JitFrames as `JITFRAME_FIXED_SIZE + - // frame_depth`; wasm CA frames follow the same per-token depth instead - // of inheriting the host arena's call-area floor. + // Every geometry retains its tail call area for later bridges. CA + // callee frames use only the homes prefix (`ca_frame_bytes`) below. + let raw_frame_value_slots = codegen::frame_value_slots(inputargs, ops); + let raw_num_ref_homes = codegen::count_ref_homes(inputargs, ops); let frame = codegen::FrameGeometry::compact( - codegen::frame_value_slots(inputargs, ops).max(FROZEN_CHAIN_VALUE_SLOTS), - codegen::count_ref_homes(inputargs, ops).max(FROZEN_CHAIN_REF_HOMES), + raw_frame_value_slots.max(FROZEN_CHAIN_VALUE_SLOTS), + raw_num_ref_homes.max(FROZEN_CHAIN_REF_HOMES), ); // Exit indices come from the global fail-index space so a cross-trace // chain's `frame[0]` resolves regardless of which module wrote it @@ -1231,6 +1295,7 @@ impl majit_backend::Backend for WasmBackend { // Loops do not emit the CA arm, but they can run on a // nursery CA frame and must reload local 0 after a collect. ca_reload_fn_ptr: wasm_jit_ca_reload_frame as *const () as usize as i64, + jf_top_addr: jf_top_addr(), ..codegen::CaParams::default() }, )?; @@ -1389,6 +1454,7 @@ impl majit_backend::Backend for WasmBackend { max_output_slots, num_ref_homes, frame, + has_trampoline_calls: std::cell::Cell::new(has_trampoline_calls), bridge_cells_base, num_guard_cells: guard_exits.len(), has_preamble, @@ -1443,7 +1509,6 @@ impl majit_backend::Backend for WasmBackend { // and `previous_tokens` are unused. let ops_owned: Vec = ops.iter().map(|rc| (**rc).clone()).collect(); let ops: &[Op] = &ops_owned; - diag_bump(0); // compile_bridge entered // is_loop=false: a bridge's terminal JUMP with no LABEL is a loop-closing @@ -1454,12 +1519,12 @@ impl majit_backend::Backend for WasmBackend { // The CA arm must be able to complete a callee deopt; without the // registered `wasm_ca_resume_deopt` slot it could not, so decline the // lift (the host round-trip path still handles the CALL_ASSEMBLER). - let allow_ca = ca_deopt_helper_slot() != 0 + let ca_candidate = ca_deopt_helper_slot() != 0 && bridge_is_self_recursive_int_ca(ops, original_token.number); - if let Some(reason) = wasm_unsupported_trace_reason(ops, false, allow_ca) { - diag_bump(1); // declined: CALL_ASSEMBLER - return Err(BackendError::Unsupported(reason)); - } + // The CA candidate's `CallAssemblerR` is a dedicated direct arm; all + // other ops are scanned against their normal emission paths. + let bridge_has_trampoline_calls = + codegen::has_trampoline_calls(inputargs, ops, ca_candidate); // Decline exception-resume bridges (`GuardException`): the guarded call // raised, so the bridge resumes into the exception handler by re-entering @@ -1504,7 +1569,7 @@ impl majit_backend::Backend for WasmBackend { source_loop_finish_fi, source_compiled_ptr, source_ca_active, - _source_has_bridges, + source_has_trampoline_calls, ) = { let source_loop = original_token .compiled @@ -1576,7 +1641,7 @@ impl majit_backend::Backend for WasmBackend { // with a stale pointer. source_loop as *const CompiledWasmLoop as usize as u64, source_loop.ca_active.get(), - !source_loop.bridge_descr_ranges.borrow().is_empty(), + source_loop.has_trampoline_calls.get(), ) }; @@ -1602,13 +1667,27 @@ impl majit_backend::Backend for WasmBackend { // restrict it to direct loop guards. A CA-shaped bridge on a nested // guard then fails codegen's CALL_ASSEMBLER handling — a deterministic // decline. - let allow_ca = allow_ca && source_is_direct; - if !allow_ca && source_ca_active { - diag_bump(14); // declined: source recursion is CA-active + let mut allow_ca = ca_candidate && source_is_direct; + let ca_trampoline_decline = if allow_ca && source_has_trampoline_calls { + Some( + "wasm backend: self-recursive CA source token or chained bridge \ + uses the host call trampoline", + ) + } else if allow_ca && bridge_has_trampoline_calls { + Some("wasm backend: self-recursive CA bridge uses the host call trampoline") + } else { + None + }; + if ca_trampoline_decline.is_some() { + // Let the ordinary non-CA CALL_ASSEMBLER decline path retain the + // interpreter fallback, but make this soundness floor observable. + diag_bump(15); + allow_ca = false; + } + if let Some(reason) = wasm_unsupported_trace_reason(ops, false, allow_ca) { + diag_bump(1); // declined: CALL_ASSEMBLER return Err(BackendError::Unsupported( - "wasm backend: source recursion is CA-active; further bridge \ - chaining declined" - .into(), + ca_trampoline_decline.unwrap_or(reason.as_str()).to_string(), )); } @@ -1622,7 +1701,19 @@ impl majit_backend::Backend for WasmBackend { let bridge_ref_homes = codegen::count_ref_homes(inputargs, ops); if bridge_value_slots > source_frame.value_slots || bridge_ref_homes > source_frame.home_slots + || (source_ca_active && bridge_has_trampoline_calls) { + if source_ca_active && bridge_has_trampoline_calls { + // Guard exits in a CA-active token execute on movable callee + // frames. Do not chain a later bridge whose own body would + // re-enter the stale-pointer host trampoline. + diag_bump(15); + return Err(BackendError::Unsupported( + "wasm backend: CA-active source cannot chain a bridge that \ + uses the host call trampoline" + .into(), + )); + } diag_bump(4); return Err(BackendError::Unsupported(format!( "wasm backend: bridge frame needs values={bridge_value_slots}, homes={bridge_ref_homes}; \ @@ -1832,7 +1923,10 @@ impl majit_backend::Backend for WasmBackend { Box::leak(build_callee_gcmap(&source_input_types, source_frame)).as_ptr() as i64; codegen::CaParams { emit_ca: true, - callee_frame_bytes: source_frame.frame_bytes, + // `compile_bridge`'s trampoline-decline floor above guarantees + // no trampoline-lowered op executes on this movable CA callee + // frame, so its tail call area is never touched. + callee_frame_bytes: source_frame.ca_frame_bytes, loop_finish_fi: source_loop_finish_fi, deopt_helper_slot: ca_deopt_helper_slot(), source_compiled_ptr, @@ -1842,10 +1936,13 @@ impl majit_backend::Backend for WasmBackend { ca_reload_caller_fn_ptr: wasm_jit_ca_reload_caller_frame as *const () as usize as i64, callee_gcmap_ptr, + inline: ca_inline_params(source_frame.ca_frame_bytes), + jf_top_addr: jf_top_addr(), } } else { codegen::CaParams { ca_reload_fn_ptr: wasm_jit_ca_reload_frame as *const () as usize as i64, + jf_top_addr: jf_top_addr(), ..codegen::CaParams::default() } }; @@ -1939,6 +2036,7 @@ impl majit_backend::Backend for WasmBackend { count, )); } + source_loop.record_chained_bridge_trampoline_calls(bridge_has_trampoline_calls); // Publish this bridge's own guard-dispatch metadata so a hot guard // INSIDE it can chain a nested sub-bridge (same resolution the // loop's own guards get, keyed by this bridge's trace_id). @@ -2177,24 +2275,10 @@ impl majit_backend::Backend for WasmBackend { .downcast_ref::() .expect("not CompiledWasmLoop"); - // Allocate frame area large enough for slots + call trampoline area + - // the Ref-home region. MIN_FRAME_BYTES accommodates the call area at - // offset 2000+; the Ref-home region (`codegen::HOME_SLOT_BASE`) follows - // it, one slot per Ref value live across a collecting call - // (`num_ref_homes`). - let min_slots = codegen::MIN_FRAME_BYTES / 8; - let base_slots = min_slots.max(1 + compiled.max_output_slots.max(compiled.num_inputs)); - // +1 for the resume-at-LABEL dispatch-key slot (codegen::DISPATCH_KEY_OFS - // = MIN_FRAME_BYTES, slot `min_slots`), which sits between the call area - // and the Ref-home region (now at HOME_SLOT_BASE = MIN_FRAME_BYTES + 8). - // `vec![0i64]` zeroes it, so a fresh host entry reads key 0 (preamble). - // - // The host/arena allocation deliberately retains its historical floor: - // any token may run here. Generated code, however, addresses only the - // compact offsets frozen on `compiled.frame`; all chained bridges use - // that same geometry or are declined at compile time. - let _ = base_slots; - let frame_size = min_slots.max((compiled.frame.frame_bytes as usize).div_ceil(8)); + // Host entry allocates the complete frozen geometry, including the tail + // call area. Chained bridges share these exact offsets; only CA callee + // frames use the smaller homes prefix (`ca_frame_bytes`). + let frame_size = (compiled.frame.frame_bytes as usize).div_ceil(8); #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { let _ = (frame_size, args); diff --git a/majit/majit-gc/src/collector.rs b/majit/majit-gc/src/collector.rs index 0de40bb473f..6c2f475cc0c 100644 --- a/majit/majit-gc/src/collector.rs +++ b/majit/majit-gc/src/collector.rs @@ -3205,6 +3205,10 @@ impl GcAllocator for MiniMarkGC { self.is_valid_gc_object(addr) && (self.nursery.contains(addr) || self.oldgen.contains(addr)) } + fn is_nursery_object(&self, addr: usize) -> bool { + self.is_nursery_object_start(addr) + } + fn write_barrier(&mut self, obj: GcRef) { self.do_write_barrier(obj); } diff --git a/majit/majit-gc/src/lib.rs b/majit/majit-gc/src/lib.rs index ec621dd241d..5a6151567a4 100644 --- a/majit/majit-gc/src/lib.rs +++ b/majit/majit-gc/src/lib.rs @@ -323,6 +323,14 @@ pub trait GcAllocator: Send { false } + /// Whether `addr` is a live object in the moving nursery. Custom trace + /// hooks use this to distinguish a nursery child whose own type walker + /// will scan its items from an old-gen child whose owner must scan an + /// off-barrier payload during a minor collection. + fn is_nursery_object(&self, _addr: usize) -> bool { + false + } + /// Current nursery free pointer. fn nursery_free(&self) -> *mut u8; @@ -734,6 +742,9 @@ impl GcAllocator for GcHandle { fn is_managed_heap_object(&self, addr: usize) -> bool { gc_sync::gc_query_reentrant(|gc| gc.is_managed_heap_object(addr)) } + fn is_nursery_object(&self, addr: usize) -> bool { + gc_sync::gc_query_reentrant(|gc| gc.is_nursery_object(addr)) + } fn nursery_free(&self) -> *mut u8 { gc_sync::gc_query_reentrant(|gc| gc.nursery_free()) } @@ -1395,14 +1406,21 @@ pub fn jitframe_shadow_stack_empty() -> bool { /// sweeps them) and fall through to `std::alloc::dealloc` for /// `std::alloc`-allocated ones. pub type GcOwnsObjectFn = fn(addr: usize) -> bool; +pub type GcIsNurseryObjectFn = fn(addr: usize) -> bool; global_hook!(static ACTIVE_GC_OWNS_OBJECT: GcOwnsObjectFn); +global_hook!(static ACTIVE_GC_IS_NURSERY_OBJECT: GcIsNurseryObjectFn); /// Install the active backend's `is_managed_heap_object` trampoline. pub fn set_active_gc_owns_object(hook: Option) { ACTIVE_GC_OWNS_OBJECT.set(hook); } +/// Install the active backend's nursery-membership predicate. +pub fn set_active_gc_is_nursery_object(hook: Option) { + ACTIVE_GC_IS_NURSERY_OBJECT.set(hook); +} + /// minimark.py:1900-1915 `id_or_identityhash` hook. pub type GcIdOrIdentityHashFn = fn(addr: usize) -> usize; @@ -1432,6 +1450,17 @@ pub fn gc_owns_object(addr: usize) -> bool { } } +/// Whether `addr` is a live object in the active backend's nursery. +pub fn gc_is_nursery_object(addr: usize) -> bool { + match ACTIVE_GC_IS_NURSERY_OBJECT.get() { + Some(f) => f(addr), + None if gc_sync::is_initialized() => { + gc_sync::gc_query_reentrant(|gc| gc.is_nursery_object(addr)) + } + None => false, + } +} + /// Return the current address for a managed object without treating it as a /// root. During a minor collection this follows an already-installed nursery /// forwarding pointer; otherwise it returns `addr` unchanged. diff --git a/majit/majit-gc/src/shadow_stack.rs b/majit/majit-gc/src/shadow_stack.rs index bd219e7f896..f97fcd2b17b 100644 --- a/majit/majit-gc/src/shadow_stack.rs +++ b/majit/majit-gc/src/shadow_stack.rs @@ -60,6 +60,9 @@ struct JitFrameShadowStack { /// Current top pointer. Compiled code embeds the address of this cell and /// mutates its inner usize directly with inline loads/stores. top: Cell, + /// One-past-the-end address of the current backing buffer. Compiled code + /// embeds this cell's address, never its value: `grow()` replaces `owner`. + limit: Cell, /// Current capacity in entries (each entry is two usize words). capacity: usize, owner: Option>, @@ -70,6 +73,7 @@ impl JitFrameShadowStack { Self { base: 0, top: Cell::new(0), + limit: Cell::new(0), capacity: 0, owner: None, } @@ -110,12 +114,17 @@ impl JitFrameShadowStack { self.base = new_ptr; self.top.set(new_ptr + used_bytes); self.capacity = new_capacity; + self.limit.set(new_ptr + new_capacity * 2 * WORD); self.owner = Some(new_buf); } fn top_addr(&self) -> usize { self.top.as_ptr() as usize } + + fn limit_addr(&self) -> usize { + self.limit.as_ptr() as usize + } } /// Callback type for tracing a libc-allocated jitframe's interior. @@ -184,6 +193,19 @@ pub fn get_root_stack_top_addr() -> usize { }) } +/// Address of this thread's grow-synchronised root-stack limit cell. +/// +/// Compiled code loads the value for every inline push. `grow()` updates it +/// after replacing the backing buffer, while the cell address itself remains +/// stable for the lifetime of the thread-local shadow-stack object. +pub fn get_root_stack_limit_addr() -> usize { + JF_ROOT_STACK.with(|stack| { + let mut stack = stack.borrow_mut(); + stack.ensure_init(); + stack.limit_addr() + }) +} + thread_local! { /// shadowstack.py:287 `root_stack_depth`. Growable via /// `increase_root_stack_depth`; can never shrink. @@ -1196,6 +1218,8 @@ mod tests { push_jf(GcRef(0xDEADBEEF)); let base_before = jf_root_stack_base_for_test(); let capacity_before = jf_root_stack_capacity_for_test(); + let limit_addr = get_root_stack_limit_addr(); + let limit_before = unsafe { *(limit_addr as *const usize) }; assert_eq!(capacity_before, DEFAULT_SHADOW_STACK_DEPTH); // Grow to 2x the default. RPython's resize copies the used @@ -1205,6 +1229,13 @@ mod tests { let base_after = jf_root_stack_base_for_test(); let capacity_after = jf_root_stack_capacity_for_test(); + assert_eq!(get_root_stack_limit_addr(), limit_addr); + assert_ne!(unsafe { *(limit_addr as *const usize) }, limit_before); + assert_eq!( + unsafe { *(limit_addr as *const usize) }, + base_after + new_cap * 2 * WORD, + "the stable limit cell must track the reallocated backing buffer" + ); assert_ne!( base_before, base_after, "resize must reallocate the backing buffer" diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.py b/pyre/bench/synth/wasm_ca_trampoline_decline.py new file mode 100644 index 00000000000..32cd7ac5c9f --- /dev/null +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.py @@ -0,0 +1,27 @@ +# Regression for wasm CA frames: this self-recursive body reaches the raw +# float-power residual call (CallF, hence the host jit_call trampoline) and +# allocates a string at every recursive level. Once the recursion bridge is +# hot, CA must be declined: a moving nursery callee frame cannot survive the +# trampoline retaining its pre-call frame pointer. + + +def descend(n): + if n < 2: + return n + allocated = str(n) * 512 + powered = (n + 1.25) ** 1.5 + # Preserve the ordinary fib recursion shape while retaining `powered` in + # the trace. `int(powered)` allocates a boxed result on each level. + return len(allocated) + descend(n - 1) + descend(n - 2) + int(powered) - int(powered) + + +def run(): + total = 0 + i = 0 + while i < 240: + total += descend(10) + i += 1 + return total + + +print(run()) diff --git a/pyre/pyre-interpreter/src/call.rs b/pyre/pyre-interpreter/src/call.rs index 46b7653823c..28952f1a89e 100644 --- a/pyre/pyre-interpreter/src/call.rs +++ b/pyre/pyre-interpreter/src/call.rs @@ -521,6 +521,7 @@ fn call_user_function_with_eval( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); return gen_frame.into_generator(); } @@ -533,6 +534,7 @@ fn call_user_function_with_eval( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); func_frame.fix_array_ptrs(); let _caller_locals_root = FrameLocalsRoot::new(frame); @@ -571,6 +573,7 @@ pub fn call_user_function_resolved( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); return gen_frame.into_generator(); } @@ -585,6 +588,7 @@ pub fn call_user_function_resolved( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); func_frame.fix_array_ptrs(); let _caller_locals_root = FrameLocalsRoot::new(frame); @@ -1060,6 +1064,7 @@ pub fn call_user_function_plain_with_ctx( w_globals, execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); return gen_frame.into_generator(); } @@ -1072,6 +1077,7 @@ pub fn call_user_function_plain_with_ctx( w_globals, execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); func_frame.fix_array_ptrs(); let _callee_locals_root = FrameLocalsRoot::new_mut(&mut func_frame); @@ -1973,6 +1979,7 @@ pub fn call_with_kwargs( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?, ); func_frame.fix_array_ptrs(); @@ -2481,6 +2488,7 @@ fn call_user_function_with_args(func: PyObjectRef, args: &[PyObjectRef]) -> PyOb w_globals, exec_ctx, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { Ok(f) => f, Err(e) => { @@ -2506,6 +2514,7 @@ fn call_user_function_with_args(func: PyObjectRef, args: &[PyObjectRef]) -> PyOb w_globals, exec_ctx, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { Ok(f) => f, Err(e) => { @@ -2549,7 +2558,13 @@ fn call_user_function_resolved_frameless(func: PyObjectRef, args: &[PyObjectRef] let mut frame = crate::pyframe::FrameBox::new(PyFrame::new_for_call_with_closure_and_globals_obj( - w_code, args, globals, w_globals, exec_ctx, closure, + w_code, + args, + globals, + w_globals, + exec_ctx, + closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )); frame.fix_array_ptrs(); if crate::pyframe::code_flags_make_generator(code_ref.flags) { @@ -3124,6 +3139,7 @@ fn build_class_inner( w_globals, exec_ctx, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, )?); // The class body executes against a namespace OBJECT (setdictscope) // so STORE_NAME / LOAD_NAME route through the object form, not the raw diff --git a/pyre/pyre-interpreter/src/eval.rs b/pyre/pyre-interpreter/src/eval.rs index a5468c74e5f..5d7b02a94c9 100644 --- a/pyre/pyre-interpreter/src/eval.rs +++ b/pyre/pyre-interpreter/src/eval.rs @@ -631,11 +631,25 @@ pub unsafe fn walk_pyframe_roots_area( // `locals()` dict, or an `exec` mapping), so forwarding the // object pointer keeps the whole namespace reachable. if !(*frame).debugdata.is_null() { + if pyre_object::gc_hook::try_gc_owns_object((*frame).debugdata as *mut u8) { + let debugdata_slot = + &mut (*frame).debugdata as *mut *mut crate::pyframe::FrameDebugData; + visitor(&mut *(debugdata_slot as *mut majit_ir::GcRef)); + } let d = &mut *(*frame).debugdata; let w_locals_slot = &mut d.w_locals as *mut PyObjectRef; visitor(&mut *(w_locals_slot as *mut majit_ir::GcRef)); let w_f_trace_slot = &mut d.w_f_trace as *mut PyObjectRef; visitor(&mut *(w_f_trace_slot as *mut majit_ir::GcRef)); + let hidden_operationerr_slot = &mut d.hidden_operationerr as *mut PyObjectRef; + visitor(&mut *(hidden_operationerr_slot as *mut majit_ir::GcRef)); + } + if !(*frame).lastblock.is_null() + && pyre_object::gc_hook::try_gc_owns_object((*frame).lastblock as *mut u8) + { + let lastblock_slot = + &mut (*frame).lastblock as *mut *mut crate::pyframe::FrameBlock; + visitor(&mut *(lastblock_slot as *mut majit_ir::GcRef)); } // pyframe.py:49 `self.w_globals` is the dict OBJECT. Its slot // was forwarded above (before the debugdata walk), so this @@ -856,11 +870,24 @@ pub fn walk_suspended_generator_frame( visitor(&mut *(w_builtin_slot as *mut majit_ir::GcRef)); if !(*frame).debugdata.is_null() { + if pyre_object::gc_hook::try_gc_owns_object((*frame).debugdata as *mut u8) { + let debugdata_slot = + &mut (*frame).debugdata as *mut *mut crate::pyframe::FrameDebugData; + visitor(&mut *(debugdata_slot as *mut majit_ir::GcRef)); + } let d = &mut *(*frame).debugdata; let w_locals_slot = &mut d.w_locals as *mut PyObjectRef; visitor(&mut *(w_locals_slot as *mut majit_ir::GcRef)); let w_f_trace_slot = &mut d.w_f_trace as *mut PyObjectRef; visitor(&mut *(w_f_trace_slot as *mut majit_ir::GcRef)); + let hidden_operationerr_slot = &mut d.hidden_operationerr as *mut PyObjectRef; + visitor(&mut *(hidden_operationerr_slot as *mut majit_ir::GcRef)); + } + if !(*frame).lastblock.is_null() + && pyre_object::gc_hook::try_gc_owns_object((*frame).lastblock as *mut u8) + { + let lastblock_slot = &mut (*frame).lastblock as *mut *mut crate::pyframe::FrameBlock; + visitor(&mut *(lastblock_slot as *mut majit_ir::GcRef)); } } } diff --git a/pyre/pyre-interpreter/src/function.rs b/pyre/pyre-interpreter/src/function.rs index 64b4cd7a158..ca336075096 100644 --- a/pyre/pyre-interpreter/src/function.rs +++ b/pyre/pyre-interpreter/src/function.rs @@ -2300,6 +2300,7 @@ fn _flat_pycall( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { Ok(f) => f, Err(e) => { @@ -2371,6 +2372,7 @@ fn _flat_pycall_defaults( w_globals, frame.execution_context, closure, + crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { Ok(f) => f, Err(e) => { diff --git a/pyre/pyre-interpreter/src/pyframe.rs b/pyre/pyre-interpreter/src/pyframe.rs index d35dd37b42f..834522803bd 100644 --- a/pyre/pyre-interpreter/src/pyframe.rs +++ b/pyre/pyre-interpreter/src/pyframe.rs @@ -164,6 +164,13 @@ pub struct PyFrame { /// drift panics on startup. pub const PYFRAME_GC_TYPE_ID: u32 = 37; +/// GC type ids appended after the existing runtime registration census. +/// `FrameDebugData` is stationary old-gen for a GC-owned frame; block-stack +/// nodes are ordinary nursery objects. Keep these at the tail of +/// `pyre-jit::eval::build_gc` so older type ids never shift. +pub const FRAME_DEBUG_DATA_GC_TYPE_ID: u32 = 103; +pub const FRAME_BLOCK_GC_TYPE_ID: u32 = 104; + /// GC header size in bytes — single source of truth is /// [`majit_gc::header::GcHeader::SIZE`]. Every `FixedObjectArray` and /// `PyFrame` allocation prepends this many zero bytes so RPython-style @@ -172,6 +179,15 @@ pub const PYFRAME_GC_TYPE_ID: u32 = 37; /// allocations route through [`majit_gc::header::alloc_with_gc_header`]. pub const GC_HEADER_SIZE: usize = majit_gc::header::GcHeader::SIZE; +/// Ownership selected by the caller that decides a frame's lifetime. +/// `FrameBox::new` call frames use `OldGenGc`; tracer-private snapshots use +/// `StdAlloc` so their locals remain valid until deterministic `Drop`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FrameLocalsArrayAllocation { + OldGenGc, + StdAlloc, +} + /// Allocation size (in bytes, including the GC header) for a /// `FixedObjectArray` of the given length. #[inline] @@ -219,6 +235,42 @@ pub unsafe fn alloc_fixed_array_with_header( } } +/// Allocate a frame locals array in the lifetime regime selected by its owner. +/// The old-gen form is the type-9 GcArray layout `[GcHeader | len | items]`. +unsafe fn alloc_frame_locals_array( + len: usize, + fill: pyre_object::PyObjectRef, + allocation: FrameLocalsArrayAllocation, +) -> *mut FixedObjectArray { + if allocation == FrameLocalsArrayAllocation::OldGenGc { + let payload = pyre_object::FIXED_ARRAY_ITEMS_OFFSET + + len * std::mem::size_of::(); + let raw = pyre_object::gc_hook::try_gc_alloc_stable_raw( + pyre_object::PY_OBJECT_ARRAY_GC_TYPE_ID, + payload, + ); + if !raw.is_null() { + let arr = raw as *mut FixedObjectArray; + unsafe { + (*arr).len = len; + let items = (*arr).items_mut_ptr(); + for i in 0..len { + items.add(i).write(fill); + } + } + return arr; + } + } + unsafe { alloc_fixed_array_with_header(len, fill) } +} + +#[inline] +fn remember_frame_locals_array(array: *mut FixedObjectArray) { + if pyre_object::gc_hook::try_gc_owns_object(array as *mut u8) { + pyre_object::gc_hook::try_gc_write_barrier(array as *mut u8); + } +} + /// Allocate a `FixedObjectArray` pre-populated from `values`. The /// resulting array has `values.len()` slots; allocation layout matches /// [`alloc_fixed_array_with_header`]. @@ -249,6 +301,7 @@ pub unsafe fn dealloc_array_with_gc_header(ptr: *mut FixedObjectArray) { if ptr.is_null() { return; } + debug_assert!(!pyre_object::gc_hook::try_gc_owns_object(ptr as *mut u8)); unsafe { let len = (*ptr).len; let raw = (ptr as *mut u8).sub(GC_HEADER_SIZE); @@ -292,9 +345,10 @@ impl FrameBox { /// GC object whose lifetime is its reachability. When the GC hook is /// installed this allocates a non-moving old-gen `PYFRAME_GC_TYPE_ID` /// block (the same `try_gc_alloc_stable` path every `W_*` uses, e.g. - /// `function.rs:373`); the block is reclaimed by a major mark-sweep - /// once no root (`walk_pyframe_roots` over the `CURRENT_FRAME` / - /// `f_backref` chain) reaches it, so `Drop` performs no manual free + /// `function.rs:373`); the collector reclaims the frame and its + /// GC-managed locals, debug data, and block stack once no root + /// (`walk_pyframe_roots` over the `CURRENT_FRAME` / `f_backref` chain) + /// reaches it, so `Drop` performs no manual free /// (`executioncontext.py:91-107 leave` frees nothing either). /// /// Before the hook is wired (bootstrap, tests) `try_gc_alloc_stable` @@ -310,6 +364,9 @@ impl FrameBox { std::mem::size_of::(), ); if !raw.is_null() { + debug_assert!(pyre_object::gc_hook::try_gc_owns_object( + frame.locals_cells_stack_w as *mut u8 + )); pyre_object::gc_interp::note_alloc(); let ptr = raw as *mut PyFrame; unsafe { @@ -322,6 +379,9 @@ impl FrameBox { pyre_object::gc_hook::try_gc_write_barrier(raw); return FrameBox { ptr }; } + debug_assert!(!pyre_object::gc_hook::try_gc_owns_object( + frame.locals_cells_stack_w as *mut u8 + )); FrameBox::new_boxed(frame) } @@ -417,11 +477,10 @@ impl Drop for FrameBox { // GC-managed (old-gen) frames are reclaimed by a major mark-sweep // when no root reaches them (`pyframe.py class PyFrame(W_Root)`; // `executioncontext.py:91-107 leave` frees nothing). Their - // `PyFrame::drop` side effects (freeing the locals array / debug - // data / block chain) run from the registered `PYFRAME_GC_TYPE_ID` - // destructor at sweep, not here. Only the `std::alloc` fallback - // box is freed manually — reconstruct and drop it, which runs - // `PyFrame::drop` for that block. + // The collector reclaims their GC-managed locals array, debug data, + // and block chain with the frame, so no manual cleanup runs here. + // Only a `std::alloc` snapshot / bootstrap fallback box is freed + // manually — reconstruct and drop it, which runs `PyFrame::drop`. if pyre_object::gc_hook::try_gc_owns_object(self.ptr as *mut u8) { return; } @@ -459,10 +518,32 @@ impl Drop for FrameLocalsRoot { } } -unsafe fn clone_debugdata_ptr(ptr: *mut FrameDebugData) -> *mut FrameDebugData { +#[inline] +fn remember_frame_debug_data(debugdata: *mut FrameDebugData) { + if pyre_object::gc_hook::try_gc_owns_object(debugdata as *mut u8) { + pyre_object::gc_hook::try_gc_write_barrier(debugdata as *mut u8); + } +} + +unsafe fn clone_debugdata_ptr( + ptr: *mut FrameDebugData, + allocation: FrameLocalsArrayAllocation, +) -> *mut FrameDebugData { unsafe { if ptr.is_null() { std::ptr::null_mut() + } else if allocation == FrameLocalsArrayAllocation::OldGenGc { + let raw = pyre_object::gc_hook::try_gc_alloc_stable_raw( + FRAME_DEBUG_DATA_GC_TYPE_ID, + std::mem::size_of::(), + ); + if !raw.is_null() { + std::ptr::write(raw as *mut FrameDebugData, (*ptr).clone()); + // The clone may carry young locals / trace callback refs. + remember_frame_debug_data(raw as *mut FrameDebugData); + return raw as *mut FrameDebugData; + } + pyre_object::lltype::malloc_raw((*ptr).clone()) } else { pyre_object::lltype::malloc_raw((*ptr).clone()) } @@ -472,30 +553,98 @@ unsafe fn clone_debugdata_ptr(ptr: *mut FrameDebugData) -> *mut FrameDebugData { unsafe fn clear_debugdata_ptr(ptr: &mut *mut FrameDebugData) { unsafe { if !(*ptr).is_null() { - drop(Box::from_raw(*ptr)); + if !pyre_object::gc_hook::try_gc_owns_object(*ptr as *mut u8) { + drop(Box::from_raw(*ptr)); + } *ptr = std::ptr::null_mut(); } } } -unsafe fn clone_block_chain(ptr: *mut FrameBlock) -> *mut FrameBlock { - unsafe { - if ptr.is_null() { - std::ptr::null_mut() - } else { - pyre_object::lltype::malloc_raw(FrameBlock { - handlerposition: (*ptr).handlerposition, - valuestackdepth: (*ptr).valuestackdepth, - previous: clone_block_chain((*ptr).previous), - }) +struct FrameBlockRoot { + slot: *mut *mut u8, + registered: bool, +} + +impl FrameBlockRoot { + unsafe fn new(block: &mut *mut FrameBlock) -> Self { + let slot = block as *mut *mut FrameBlock as *mut *mut u8; + let registered = unsafe { pyre_object::gc_hook::try_gc_add_root(slot) }; + Self { slot, registered } + } +} + +impl Drop for FrameBlockRoot { + fn drop(&mut self) { + if self.registered { + pyre_object::gc_hook::try_gc_remove_root(self.slot); } } } +unsafe fn alloc_frame_block( + block: FrameBlock, + allocation: FrameLocalsArrayAllocation, +) -> *mut FrameBlock { + if allocation == FrameLocalsArrayAllocation::OldGenGc { + if let Some(raw) = pyre_object::gc_hook::try_gc_alloc( + FRAME_BLOCK_GC_TYPE_ID, + std::mem::size_of::(), + ) + .filter(|raw| !raw.is_null()) + { + unsafe { std::ptr::write(raw as *mut FrameBlock, block) }; + return raw as *mut FrameBlock; + } + } + pyre_object::lltype::malloc_raw(block) +} + +unsafe fn clone_block_chain( + ptr: *mut FrameBlock, + allocation: FrameLocalsArrayAllocation, +) -> *mut FrameBlock { + // `previous` is assigned only while a node is constructed and points to + // a strictly older node. Rebuild oldest-to-newest for the GC regime, so + // a node allocated in old-gen can never acquire a younger predecessor and + // no write barrier is needed for `previous`. + let mut source = Vec::new(); + let mut current = ptr; + while !current.is_null() { + unsafe { + source.push(FrameBlock { + handlerposition: (*current).handlerposition, + valuestackdepth: (*current).valuestackdepth, + previous: std::ptr::null_mut(), + }); + current = (*current).previous; + } + } + + let mut cloned = std::ptr::null_mut(); + for mut block in source.into_iter().rev() { + let _root = unsafe { FrameBlockRoot::new(&mut cloned) }; + block.previous = std::ptr::null_mut(); + let node = unsafe { alloc_frame_block(block, allocation) }; + unsafe { (*node).previous = cloned }; + cloned = node; + } + cloned +} + unsafe fn clear_block_chain(ptr: &mut *mut FrameBlock) { unsafe { + if !(*ptr).is_null() && pyre_object::gc_hook::try_gc_owns_object(*ptr as *mut u8) { + // A GC chain is uniformly managed by its owning GC frame. The + // collector traces `previous` and reclaims the nodes itself. + *ptr = std::ptr::null_mut(); + return; + } let mut current = *ptr; while !current.is_null() { + debug_assert!(!pyre_object::gc_hook::try_gc_owns_object( + current as *mut u8 + )); let block = Box::from_raw(current); current = block.previous; } @@ -506,26 +655,34 @@ unsafe fn clear_block_chain(ptr: &mut *mut FrameBlock) { impl Drop for PyFrame { fn drop(&mut self) { // Reached only for a `std::alloc`-backed frame (the `FrameBox` - // fallback box, or a bare stack `PyFrame`): its `locals_cells_stack_w` - // is always a `std::alloc` array, so free it here. GC-managed frames - // never run `PyFrame::drop` (their `FrameBox::drop` returns early); - // their contents are freed by the `PYFRAME_GC_TYPE_ID` destructor. + // fallback box, or a bare stack `PyFrame`): it owns `std::alloc` + // resources, so free them here. GC-managed frames never run + // `PyFrame::drop`; the collector reclaims their frame-owned resources. unsafe { self.free_owned_contents(true) }; } } impl PyFrame { - /// Free the frame's owned off-GC resources — the `locals_cells_stack_w` - /// array, the `FrameDebugData` box, and the `FrameBlock` chain. Shared - /// by `Drop for PyFrame` (the `std::alloc` fallback path) and the - /// `PYFRAME_GC_TYPE_ID` destructor (`pyframe_object_destructor`) run when - /// a GC-managed frame is swept. + #[inline] + fn aux_allocation(&self) -> FrameLocalsArrayAllocation { + if pyre_object::gc_hook::try_gc_owns_object(self as *const Self as *mut u8) { + FrameLocalsArrayAllocation::OldGenGc + } else { + FrameLocalsArrayAllocation::StdAlloc + } + } + + /// Free the `std::alloc` resources owned by a snapshot, fallback, or bare + /// stack frame: its `locals_cells_stack_w` array, `FrameDebugData` box, + /// and `FrameBlock` chain. This is reached only through `Drop for + /// PyFrame`; GC-managed frames and all of their corresponding resources + /// are reclaimed by the collector. /// /// `free_locals_array` gates freeing `locals_cells_stack_w`: it is a - /// `std::alloc` block for every `FrameBox`/stack frame (free it), but a - /// GC-managed `PY_OBJECT_ARRAY_GC_TYPE_ID` array for a JIT-built inline - /// frame (the GC sweeps it — freeing it here would double-free). The - /// caller decides by querying `try_gc_owns_object` on the array. + /// `std::alloc` block for this Drop-only regime (free it), but can remain + /// false if a future regime-mixup supplies a GC-managed array. The + /// `try_gc_owns_object` checks in this cleanup path are retained as a + /// guard against freeing collector-owned storage. /// /// # Safety /// Runs at most once per frame — the pointers are nulled as they are @@ -653,6 +810,8 @@ impl Default for FrameDebugData { /// pyopcode.py:1875-1897 FrameBlock — linked list node for the block stack. /// `previous` forms a singly-linked list; `lastblock` in PyFrame is the head. +/// It is assigned only during construction and targets a strictly older node, +/// so an old-gen node never needs a write barrier for it. #[derive(Debug, Clone, Copy)] pub struct FrameBlock { /// pyopcode.py:1883 @@ -1320,8 +1479,27 @@ impl PyFrame { #[inline] fn getorcreate_debug_data(&mut self, init_lineno: isize) -> &mut FrameDebugData { if self.debugdata.is_null() { - self.debugdata = - pyre_object::lltype::malloc_raw(FrameDebugData::new(self.pycode, init_lineno)); + let allocation = self.aux_allocation(); + let value = FrameDebugData::new(self.pycode, init_lineno); + self.debugdata = if allocation == FrameLocalsArrayAllocation::OldGenGc { + let raw = pyre_object::gc_hook::try_gc_alloc_stable_raw( + FRAME_DEBUG_DATA_GC_TYPE_ID, + std::mem::size_of::(), + ); + if !raw.is_null() { + unsafe { std::ptr::write(raw as *mut FrameDebugData, value) }; + raw as *mut FrameDebugData + } else { + pyre_object::lltype::malloc_raw(value) + } + } else { + pyre_object::lltype::malloc_raw(value) + }; + // The allocation starts null-filled, but callers commonly seed + // it immediately with w_globals or a freshly allocated mapping. + // Remembering the completed object is harmless for Box fallback + // and keeps the old-gen debug payload visible to the next minor. + remember_frame_debug_data(self.debugdata); } unsafe { &mut *self.debugdata } } @@ -1335,7 +1513,12 @@ impl PyFrame { /// PyPy-compatible `getorcreatedebug()`. #[inline] pub fn getorcreatedebug(&mut self, init_lineno: isize) -> &mut FrameDebugData { - self.getorcreate_debug_data(init_lineno) + self.getorcreate_debug_data(init_lineno); + // Callers mutate the returned payload directly (notably w_f_trace) + // without their own barrier. Keep the old-gen payload remembered + // before exposing it for that mutation. + remember_frame_debug_data(self.debugdata); + unsafe { &mut *self.debugdata } } /// PyPy-compatible alias for `code()`. @@ -1408,6 +1591,7 @@ impl PyFrame { // observable instead of faulting. let w_locals = unsafe { pyre_object::w_dict_new() }; self.getorcreate_debug_data(-1).w_locals = w_locals; + remember_frame_debug_data(self.debugdata); w_locals } @@ -1420,14 +1604,20 @@ impl PyFrame { outer_func: PyObjectRef, ) { let _ = outer_func; + let allocation = self.aux_allocation(); self.pycode = code; let raw = unsafe { crate::w_code_get_ptr(code as pyre_object::PyObjectRef) as *const CodeObject }; - unsafe { dealloc_array_with_gc_header(self.locals_cells_stack_w) }; + if !self.locals_cells_stack_w.is_null() + && !pyre_object::gc_hook::try_gc_owns_object(self.locals_cells_stack_w as *mut u8) + { + unsafe { dealloc_array_with_gc_header(self.locals_cells_stack_w) }; + } self.locals_cells_stack_w = unsafe { - alloc_fixed_array_with_header( + alloc_frame_locals_array( (&*raw).varnames.len() + ncells(&*raw) + (&*raw).max_stackdepth as usize, PY_NULL, + allocation, ) }; self.valuestackdepth = unsafe { (&*raw).varnames.len() + ncells(&*raw) }; @@ -1465,6 +1655,7 @@ impl PyFrame { self.initialize_frame_scopes(outer_func, code).expect( "PyFrame::__init__: initialize_frame_scopes raised — caller should use createframe", ); + remember_frame_locals_array(self.locals_cells_stack_w); } /// PyPy-compatible `__repr__`. @@ -1808,7 +1999,8 @@ impl PyFrame { // the whole `trace_bytecode` walk, during which a major GC cycle can // complete; no root reaches it, so it must NOT have GC lifetime — // `new_boxed` gives it a deterministic scope-end free. - let mut frame = FrameBox::new_boxed(self.build_snapshot_frame()); + let mut frame = + FrameBox::new_boxed(self.build_snapshot_frame(FrameLocalsArrayAllocation::StdAlloc)); // fix_array_ptrs AFTER Box allocation: inline_buf ptr must // point to the heap-allocated frame, not a stale stack address. frame.fix_array_ptrs(); @@ -1823,17 +2015,25 @@ impl PyFrame { /// write during recording would leak to the real heap and double-apply /// on the compiled loop's re-run; Gap 10 removed that path (inline-frame /// STORE_GLOBAL records as deferred IR, applied exactly once). - fn build_snapshot_frame(&self) -> PyFrame { + fn build_snapshot_frame(&self, allocation: FrameLocalsArrayAllocation) -> PyFrame { PyFrame { ob_header: frame_ob_header(), execution_context: self.execution_context, pycode: self.pycode, - locals_cells_stack_w: unsafe { alloc_fixed_array_from_vec(self.locals_w().to_vec()) }, + locals_cells_stack_w: unsafe { + let values = self.locals_w().to_vec(); + let array = alloc_frame_locals_array(values.len(), PY_NULL, allocation); + for (i, value) in values.into_iter().enumerate() { + (*array).items_mut_ptr().add(i).write(value); + } + remember_frame_locals_array(array); + array + }, valuestackdepth: self.valuestackdepth, last_instr: self.last_instr, escaped: self.escaped, - debugdata: unsafe { clone_debugdata_ptr(self.debugdata) }, - lastblock: unsafe { clone_block_chain(self.lastblock) }, + debugdata: unsafe { clone_debugdata_ptr(self.debugdata, allocation) }, + lastblock: unsafe { clone_block_chain(self.lastblock, allocation) }, vable_token: self.vable_token, frame_finished_execution: self.frame_finished_execution, f_generator_nowref: self.f_generator_nowref, @@ -1850,7 +2050,8 @@ impl PyFrame { /// long as the generator object reaches it (`generator.py` holds the /// frame), and the generator's custom trace greys the frame block. pub fn snapshot_for_generator(&self) -> FrameBox { - let mut frame = FrameBox::new(self.build_snapshot_frame()); + let mut frame = + FrameBox::new(self.build_snapshot_frame(FrameLocalsArrayAllocation::OldGenGc)); frame.fix_array_ptrs(); frame } @@ -2099,8 +2300,16 @@ impl PyFrame { /// pyframe.py:186 append_block #[inline] pub fn append_block(&mut self, mut block: FrameBlock) { - block.previous = self.lastblock; - self.lastblock = pyre_object::lltype::malloc_raw(block); + let allocation = self.aux_allocation(); + let mut previous = self.lastblock; + let _root = unsafe { FrameBlockRoot::new(&mut previous) }; + block.previous = std::ptr::null_mut(); + let node = unsafe { alloc_frame_block(block, allocation) }; + // `previous` is written only here (and in clone/unpickle construction) + // and always targets the strictly older node. Thus an old-gen block + // never points to a younger block and needs no write barrier. + unsafe { (*node).previous = previous }; + self.lastblock = node; } /// pyframe.py:190 pop_block @@ -2110,11 +2319,19 @@ impl PyFrame { return None; } unsafe { - let block = Box::from_raw(self.lastblock); - self.lastblock = block.previous; - let mut result = *block; - result.previous = std::ptr::null_mut(); - Some(result) + let current = self.lastblock; + if pyre_object::gc_hook::try_gc_owns_object(current as *mut u8) { + let mut result = *current; + self.lastblock = result.previous; + result.previous = std::ptr::null_mut(); + Some(result) + } else { + let block = Box::from_raw(current); + self.lastblock = block.previous; + let mut result = *block; + result.previous = std::ptr::null_mut(); + Some(result) + } } } @@ -2953,6 +3170,7 @@ impl PyFrame { w_globals, execution_context, PY_NULL, + FrameLocalsArrayAllocation::StdAlloc, ) } @@ -2976,6 +3194,7 @@ impl PyFrame { w_globals, execution_context, closure, + FrameLocalsArrayAllocation::StdAlloc, ) } @@ -2990,6 +3209,7 @@ impl PyFrame { w_globals: PyObjectRef, execution_context: *const PyExecutionContext, closure: PyObjectRef, + allocation: FrameLocalsArrayAllocation, ) -> Result { let w_builtin = if w_globals.is_null() { crate::baseobjspace::frame_builtin(globals, execution_context) @@ -3003,6 +3223,7 @@ impl PyFrame { execution_context, closure, w_builtin, + allocation, )) } @@ -3024,6 +3245,7 @@ impl PyFrame { w_globals: PyObjectRef, execution_context: *const PyExecutionContext, closure: PyObjectRef, + allocation: FrameLocalsArrayAllocation, ) -> Self { let w_builtin = if w_globals.is_null() { crate::baseobjspace::frame_builtin(globals, execution_context) @@ -3037,6 +3259,7 @@ impl PyFrame { execution_context, closure, w_builtin, + allocation, ) } @@ -3050,6 +3273,7 @@ impl PyFrame { execution_context: *const PyExecutionContext, closure: PyObjectRef, w_builtin: PyObjectRef, + allocation: FrameLocalsArrayAllocation, ) -> Self { let code_ref = unsafe { &*(crate::w_code_get_ptr(code as pyre_object::PyObjectRef) as *const CodeObject) @@ -3058,8 +3282,9 @@ impl PyFrame { let num_cells = ncells(code_ref); let max_stack = code_ref.max_stackdepth as usize; - let locals_cells_stack_w = - unsafe { alloc_fixed_array_with_header(num_locals + num_cells + max_stack, PY_NULL) }; + let locals_cells_stack_w = unsafe { + alloc_frame_locals_array(num_locals + num_cells + max_stack, PY_NULL, allocation) + }; { // Populate the freshly-allocated array via its mutable slice. @@ -3091,6 +3316,11 @@ impl PyFrame { } } + // Stable frame-locals arrays are filled before their owning frame is + // published. `w_cell_new` uses the non-collecting old-gen allocator; + // remember the completed array before the next allocating operation. + remember_frame_locals_array(locals_cells_stack_w); + // pyframe.py:103 — stamp `pycode.w_globals`; side effect only (the // gated debugdata snapshot retired in favour of `w_globals`). unsafe { @@ -3418,7 +3648,9 @@ pub fn createframe_obj( ob_header: frame_ob_header(), execution_context, pycode: code, - locals_cells_stack_w: unsafe { alloc_fixed_array_with_header(size, PY_NULL) }, + locals_cells_stack_w: unsafe { + alloc_frame_locals_array(size, PY_NULL, FrameLocalsArrayAllocation::OldGenGc) + }, valuestackdepth: num_locals + num_cells, last_instr: -1, escaped: false, @@ -3447,6 +3679,7 @@ pub fn createframe_obj( let _root = FrameLocalsRoot::new(frame.as_mut_ptr()); frame.initialize_frame_scopes(outer_ref, code)?; } + remember_frame_locals_array(frame.locals_cells_stack_w); Ok(frame) } diff --git a/pyre/pyre-jit-trace/src/descr.rs b/pyre/pyre-jit-trace/src/descr.rs index 0f10fad67c6..0162c75a142 100644 --- a/pyre/pyre-jit-trace/src/descr.rs +++ b/pyre/pyre-jit-trace/src/descr.rs @@ -635,6 +635,10 @@ pub use pyre_object::unicodeobject::W_UNICODE_GC_TYPE_ID; // Registered ahead of any future // `NewWithVtable(PyFrame)` in trace IR. pub use pyre_interpreter::pyframe::PYFRAME_GC_TYPE_ID; +// Appended tail registrations for PyFrame-owned auxiliary objects. These live +// with their Rust layouts in pyre-interpreter and are re-exported here beside +// PYFRAME_GC_TYPE_ID for the runtime registration census. +pub use pyre_interpreter::pyframe::{FRAME_BLOCK_GC_TYPE_ID, FRAME_DEBUG_DATA_GC_TYPE_ID}; fn field_descr_from_group(group: &PyreObjectDescrGroup, index: usize) -> DescrRef { let field_descr = group diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index f805e100f2f..f888374b084 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -12775,6 +12775,7 @@ pub(crate) fn assemble_bridge_inline_pending( w_globals, execution_context, pyre_object::PY_NULL, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::StdAlloc, ); { let arr = concrete_frame.locals_w_mut(); diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 7c943c137e3..eebfc790c43 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -240,6 +240,7 @@ fn try_commit_midbody_abort( w_globals, ec, pyre_object::PY_NULL, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { Ok(frame) => frame, Err(_) => return false, diff --git a/pyre/pyre-jit-trace/src/trace_opcode.rs b/pyre/pyre-jit-trace/src/trace_opcode.rs index 528d3aaf68a..f94a55e2087 100644 --- a/pyre/pyre-jit-trace/src/trace_opcode.rs +++ b/pyre/pyre-jit-trace/src/trace_opcode.rs @@ -6818,6 +6818,7 @@ impl MIFrame { callee_globals_obj, caller_exec_ctx, closure, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::StdAlloc, )?; callee_frame.fix_array_ptrs(); diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 27fc6104c55..dc980da524f 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -726,26 +726,26 @@ unsafe fn memoryview_object_destructor(obj_addr: usize) { /// express two of those slots: the `locals_cells_stack_w` items when the /// array is a stationary `std::alloc` block (regime-a — the collector /// never enters `trace_and_update_object` on a non-nursery array so its -/// varsize walker never runs), and the `debugdata->{w_locals, w_f_trace}` -/// refs, which live one pointer indirection away inside a non-GC -/// `malloc`'d `FrameDebugData`. Both require a custom trace. +/// varsize walker never runs), and the in-place scan of old-gen / Box +/// `FrameDebugData` fields. Both require a custom trace. /// /// Forwarded (mirrors `walk_pyframe_roots` eval.rs:496-556): /// - `f_backref` — the parent frame pointer. /// - `pycode` — visited to match the walker; inert while code objects /// are Box-immortal (`is_nursery_object_start` short-circuits). -/// - `locals_cells_stack_w` — the array pointer. When the array is a -/// GC-managed (moving) nursery block its slot is forwarded and the -/// type-9 varsize walker forwards the items; when it is a stationary -/// `std::alloc` block each item is forwarded in place — the same -/// regime split as `list_object_custom_trace` / `tuple_object_custom_trace`. +/// - `locals_cells_stack_w` — the array pointer. A GC-managed nursery +/// block forwards through its field slot and its type-9 walker owns the +/// items. An old-gen GC block also visits the field slot and walks its +/// items in place: barrier-less interpreter stores require that at minors, +/// and it is harmless duplicate marking at majors. A stationary +/// `std::alloc` block always forwards its items in place. /// - `f_generator_nowref`, `w_yielding_from`, `w_builtin`, `w_globals` /// — the ref-bearing statics. -/// - `debugdata->w_locals`, `debugdata->w_f_trace` — null-guarded. +/// - `debugdata` / `lastblock` — managed field slots are forwarded. +/// - `debugdata->{w_locals, w_f_trace, hidden_operationerr}` — null-guarded. /// /// Excluded (matches the walker): `execution_context` (persistent, not -/// GC), `debugdata`/`lastblock` (non-GC heap), the module-dict / method- -/// cache / prebuilt-family global walks (those are not frame-owned; the +/// GC), the module-dict / method-cache / prebuilt-family global walks (those are not frame-owned; the /// root walker performs them once per collection, not per frame). unsafe fn pyframe_object_custom_trace(obj_addr: usize, f: &mut dyn FnMut(*mut majit_ir::GcRef)) { let frame = unsafe { &mut *(obj_addr as *mut PyFrame) }; @@ -753,23 +753,32 @@ unsafe fn pyframe_object_custom_trace(obj_addr: usize, f: &mut dyn FnMut(*mut ma f(&mut frame.f_backref as *mut *mut PyFrame as *mut majit_ir::GcRef); f(&mut frame.pycode as *mut *const () as *mut majit_ir::GcRef); - // locals_cells_stack_w: forward the array pointer, then its items. + // locals_cells_stack_w: visit the field slot for every GC array so major + // marking reaches it. A nursery array is subsequently scanned by its own + // type-9 walker; an old-gen array also needs this in-place scan because + // interpreter stores do not write-barrier its items. At a major, its own + // walker reaches them too, so this is harmless duplicate marking. + // RPython's phase-agnostic precedent is jitframe.py:104 `jitframe_trace`. let array = frame.locals_cells_stack_w; if !array.is_null() { - if pyre_object::gc_hook::try_gc_owns_object(array as *mut u8) { - // GC-managed (moving) nursery block: hand the collector the - // array field slot; the type-9 varsize walker forwards the - // items once the array itself is copied. + let walk_items = if pyre_object::gc_hook::try_gc_owns_object(array as *mut u8) { f( &mut frame.locals_cells_stack_w as *mut *mut pyre_object::FixedObjectArray as *mut majit_ir::GcRef, ); + !majit_gc::gc_is_nursery_object(array as usize) } else { - // Stationary `std::alloc` block (type_id 0, never entered by - // `trace_and_update_object`): forward each item in place. Walk - // the FULL fixed-length array, not just the live prefix — - // matching `walk_pyframe_roots` (eval.rs:626), which forwards - // popped-in-transit argument slots past `valuestackdepth`. + true + }; + // Stationary `std::alloc` blocks (never entered by + // `trace_and_update_object`) and old-gen GC blocks forward the FULL + // fixed-length array, not just the live prefix. The old-gen major + // walk is idempotent duplicate marking; the minor walk covers + // barrier-less interpreter stores. This matches RPython's + // phase-agnostic jitframe.py:104 `jitframe_trace`. + // This matches `walk_pyframe_roots` (eval.rs:626), which forwards + // popped-in-transit argument slots past `valuestackdepth`. + if walk_items { let arr = unsafe { &mut *array }; let base = arr.items_mut_ptr(); for i in 0..arr.len() { @@ -784,34 +793,42 @@ unsafe fn pyframe_object_custom_trace(obj_addr: usize, f: &mut dyn FnMut(*mut ma f(&mut frame.w_globals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); if !frame.debugdata.is_null() { - let d = unsafe { &mut *frame.debugdata }; - f(&mut d.w_locals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); - f(&mut d.w_f_trace as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); + let debugdata = frame.debugdata; + let walk_fields = if pyre_object::gc_hook::try_gc_owns_object(debugdata as *mut u8) { + f( + &mut frame.debugdata as *mut *mut pyre_interpreter::pyframe::FrameDebugData + as *mut majit_ir::GcRef, + ); + !majit_gc::gc_is_nursery_object(debugdata as usize) + } else { + true + }; + // A nursery payload is scanned by FRAME_DEBUG_DATA_GC_TYPE_ID's + // ordinary offset walker. Box payloads and old-gen payloads need this + // in-place walk because interpreter stores do not individually + // write-barrier w_f_trace and its sibling fields. The old-gen major + // walk is harmless duplicate marking, matching RPython's + // phase-agnostic jitframe.py:104 `jitframe_trace` contract. + if walk_fields { + let d = unsafe { &mut *frame.debugdata }; + f(&mut d.w_locals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); + f(&mut d.w_f_trace as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); + f(&mut d.hidden_operationerr as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); + } + } + + if !frame.lastblock.is_null() + && pyre_object::gc_hook::try_gc_owns_object(frame.lastblock as *mut u8) + { + // FRAME_BLOCK_GC_TYPE_ID's `previous` walker forwards the rest of + // the chain. Blocks themselves contain no PyObjectRefs. + f( + &mut frame.lastblock as *mut *mut pyre_interpreter::pyframe::FrameBlock + as *mut majit_ir::GcRef, + ); } } -/// Destructor for `PyFrame` (type id [`PYFRAME_GC_TYPE_ID`]). -/// -/// A GC-managed frame is reclaimed by a major mark-sweep once no root -/// reaches it (`pyframe.py class PyFrame(W_Root)`; `FrameBox::drop` -/// performs no manual free for these). This runs at sweep to release the -/// frame's owned off-GC resources — the `FrameDebugData` box, the -/// `FrameBlock` chain, and the `locals_cells_stack_w` array — mirroring -/// `PyFrame::drop`, which handles the `std::alloc` fallback frames instead. -/// -/// The locals array is freed only when it is a `std::alloc` block (every -/// `FrameBox` frame): a JIT-built inline frame's array is a GC-managed -/// `PY_OBJECT_ARRAY_GC_TYPE_ID` block the collector sweeps on its own, so -/// freeing it here would double-free. `try_gc_owns_object` distinguishes -/// them — the same regime split `pyframe_object_custom_trace` uses. -unsafe fn pyframe_object_destructor(obj_addr: usize) { - let frame = unsafe { &mut *(obj_addr as *mut PyFrame) }; - let array = frame.locals_cells_stack_w; - let free_locals_array = - !array.is_null() && !pyre_object::gc_hook::try_gc_owns_object(array as *mut u8); - unsafe { frame.free_owned_contents(free_locals_array) }; -} - /// RPython jitexc.py:53 ContinueRunningNormally parity. pub(crate) enum LoopResult { Done(PyResult), @@ -827,18 +844,18 @@ enum JitAction { } use crate::jit::descr::{ - BUILTIN_CODE_GC_TYPE_ID, FUNCTION_GC_TYPE_ID, GC_FLOAT_ARRAY_GC_TYPE_ID, - GC_INT_ARRAY_GC_TYPE_ID, JITFRAME_GC_TYPE_ID, OBJECT_GC_TYPE_ID, PY_OBJECT_ARRAY_GC_TYPE_ID, - PYFRAME_GC_TYPE_ID, RANGE_ITER_GC_TYPE_ID, SPECIALISED_TUPLE_FF_GC_TYPE_ID, - SPECIALISED_TUPLE_II_GC_TYPE_ID, SPECIALISED_TUPLE_OO_GC_TYPE_ID, VREF_GC_TYPE_ID, - W_BASE_EXCEPTION_GC_TYPE_ID, W_BOOL_GC_TYPE_ID, W_BYTEARRAY_GC_TYPE_ID, W_BYTES_GC_TYPE_ID, - W_CELL_GC_TYPE_ID, W_CLASSMETHOD_GC_TYPE_ID, W_COUNT_GC_TYPE_ID, W_DICT_GC_TYPE_ID, - W_DICT_PROXY_GC_TYPE_ID, W_FLOAT_GC_TYPE_ID, W_GENERATOR_GC_TYPE_ID, W_INT_GC_TYPE_ID, - W_LIST_GC_TYPE_ID, W_LONG_GC_TYPE_ID, W_MEMBER_GC_TYPE_ID, W_METHOD_GC_TYPE_ID, - W_MODULE_DICT_GC_TYPE_ID, W_MODULE_GC_TYPE_ID, W_PROPERTY_GC_TYPE_ID, W_REPEAT_GC_TYPE_ID, - W_SEQ_ITER_GC_TYPE_ID, W_SET_GC_TYPE_ID, W_SLICE_GC_TYPE_ID, W_STATICMETHOD_GC_TYPE_ID, - W_SUPER_GC_TYPE_ID, W_TUPLE_GC_TYPE_ID, W_TYPE_GC_TYPE_ID, W_UNICODE_GC_TYPE_ID, - W_UNION_GC_TYPE_ID, + BUILTIN_CODE_GC_TYPE_ID, FRAME_BLOCK_GC_TYPE_ID, FRAME_DEBUG_DATA_GC_TYPE_ID, + FUNCTION_GC_TYPE_ID, GC_FLOAT_ARRAY_GC_TYPE_ID, GC_INT_ARRAY_GC_TYPE_ID, JITFRAME_GC_TYPE_ID, + OBJECT_GC_TYPE_ID, PY_OBJECT_ARRAY_GC_TYPE_ID, PYFRAME_GC_TYPE_ID, RANGE_ITER_GC_TYPE_ID, + SPECIALISED_TUPLE_FF_GC_TYPE_ID, SPECIALISED_TUPLE_II_GC_TYPE_ID, + SPECIALISED_TUPLE_OO_GC_TYPE_ID, VREF_GC_TYPE_ID, W_BASE_EXCEPTION_GC_TYPE_ID, + W_BOOL_GC_TYPE_ID, W_BYTEARRAY_GC_TYPE_ID, W_BYTES_GC_TYPE_ID, W_CELL_GC_TYPE_ID, + W_CLASSMETHOD_GC_TYPE_ID, W_COUNT_GC_TYPE_ID, W_DICT_GC_TYPE_ID, W_DICT_PROXY_GC_TYPE_ID, + W_FLOAT_GC_TYPE_ID, W_GENERATOR_GC_TYPE_ID, W_INT_GC_TYPE_ID, W_LIST_GC_TYPE_ID, + W_LONG_GC_TYPE_ID, W_MEMBER_GC_TYPE_ID, W_METHOD_GC_TYPE_ID, W_MODULE_DICT_GC_TYPE_ID, + W_MODULE_GC_TYPE_ID, W_PROPERTY_GC_TYPE_ID, W_REPEAT_GC_TYPE_ID, W_SEQ_ITER_GC_TYPE_ID, + W_SET_GC_TYPE_ID, W_SLICE_GC_TYPE_ID, W_STATICMETHOD_GC_TYPE_ID, W_SUPER_GC_TYPE_ID, + W_TUPLE_GC_TYPE_ID, W_TYPE_GC_TYPE_ID, W_UNICODE_GC_TYPE_ID, W_UNION_GC_TYPE_ID, }; use majit_gc::collector::MiniMarkGC; use majit_metainterp::JitDriver; @@ -1609,19 +1626,15 @@ fn build_gc() -> Box { // remain `type_id = 0` off-GC blocks reached only as roots via // `walk_jit_callee_frame_roots` (S2c). // - // The destructor releases a swept GC-managed frame's owned off-GC - // resources (debug data, block chain, and the `std::alloc` locals - // array of a `FrameBox` frame — NOT a JIT inline frame's GC array) — - // `FrameBox::drop` no longer frees them under policy A - // (`executioncontext.py:91-107 leave` frees nothing; frame lifetime - // is GC reachability). - let pyframe_tid = gc.register_type( - majit_gc::trace::TypeInfo::with_custom_trace( - std::mem::size_of::(), - pyframe_object_custom_trace, - ) - .with_destructor_fn(pyframe_object_destructor), - ); + // Frame-owned locals arrays, debug data, and block-stack nodes are all + // GC-managed. The collector reclaims them with the frame once it is + // unreachable; `FrameBox::drop` only frees the `std::alloc` snapshot / + // bootstrap fallback regime. With no destructor or weakref flag, + // `type_alloc_is_plain` admits PYFRAME's normal allocation fast paths. + let pyframe_tid = gc.register_type(majit_gc::trace::TypeInfo::with_custom_trace( + std::mem::size_of::(), + pyframe_object_custom_trace, + )); debug_assert_eq!(pyframe_tid, PYFRAME_GC_TYPE_ID); // `W_DictProxyObject` carries a single GC-traceable // `w_mapping: PyObjectRef` slot (the wrapped W_DictObject — @@ -2302,6 +2315,31 @@ fn build_gc() -> Box { .with_external_size(pyre_object::longobject::bigint_external_size), ); pyre_object::longobject::set_bigint_gc_type_id(bigint_tid); + // PyPy's FrameDebugData is a plain GC object. It owns three PyObjectRef + // fields; once the frame custom trace greys the payload, the ordinary + // offset walker finds all of them during a major mark. + let frame_debug_data_tid = gc.register_type(TypeInfo::with_gc_ptrs( + std::mem::size_of::(), + vec![ + std::mem::offset_of!(pyre_interpreter::pyframe::FrameDebugData, w_locals), + std::mem::offset_of!(pyre_interpreter::pyframe::FrameDebugData, w_f_trace), + std::mem::offset_of!( + pyre_interpreter::pyframe::FrameDebugData, + hidden_operationerr + ), + ], + )); + debug_assert_eq!(frame_debug_data_tid, FRAME_DEBUG_DATA_GC_TYPE_ID); + // Block-stack nodes are young GC objects. `previous` is their only GC + // edge, so the normal walker forwards and major-marks an entire chain. + let frame_block_tid = gc.register_type(TypeInfo::with_gc_ptrs( + std::mem::size_of::(), + vec![std::mem::offset_of!( + pyre_interpreter::pyframe::FrameBlock, + previous + )], + )); + debug_assert_eq!(frame_block_tid, FRAME_BLOCK_GC_TYPE_ID); // rclass.py:340-346 — assign subclassrange_{min,max} to each // vtable entry. freeze_types() runs assign_inheritance_ids // (normalizecalls.py:373-389), then we write the computed ranges diff --git a/pyre/pyre-wasm-runner/src/main.rs b/pyre/pyre-wasm-runner/src/main.rs index eb99d7344be..da0837aed8f 100644 --- a/pyre/pyre-wasm-runner/src/main.rs +++ b/pyre/pyre-wasm-runner/src/main.rs @@ -96,6 +96,9 @@ struct Host { /// guard-exit fail_index) over every host round-trip, so we can see which /// guard keeps returning to the host instead of chaining in-module. exec_hist: std::collections::BTreeMap<(u32, u32), u64>, + /// `PYRE_WASM_GUEST_PROFILE` sampling profiler; taken/restored around each + /// epoch tick so `sample` can borrow the store it lives in. + guest_profiler: Option, } fn main() { @@ -209,11 +212,48 @@ fn run(module_path: &PathBuf, source: &str) -> Result { if fuel_limit.is_some() { config.consume_fuel(true); } + // Diagnostic: PYRE_WASM_GUEST_PROFILE= writes a sampling profile + // of the guest in the Firefox processed format (main-module symbols only; + // JIT trace modules appear as omitted frames under `execute_token`). View + // at https://profiler.firefox.com/. Samples land on epoch checkpoints + // (function entries / loop back-edges), so call-dense small functions are + // over-represented and bulk ops (memory.fill) are attributed to the next + // checkpoint. Epoch interruption changes main-module codegen, so pair with + // PYRE_WASM_NO_CACHE=1 to avoid clobbering the shared .cwasm cache. + let guest_profile_out = std::env::var("PYRE_WASM_GUEST_PROFILE").ok(); + if guest_profile_out.is_some() { + config.epoch_interruption(true); + } let engine = Engine::new(&config)?; let module = load_main_module(&engine, module_path)?; let mut store = Store::new(&engine, Host::default()); + if guest_profile_out.is_some() { + const SAMPLE_INTERVAL: std::time::Duration = std::time::Duration::from_micros(200); + let profiler = wasmtime::GuestProfiler::new( + &engine, + "pyre-wasm", + SAMPLE_INTERVAL, + vec![("pyre_wasm".to_string(), module.clone())], + )?; + store.data_mut().guest_profiler = Some(profiler); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut ctx| { + if let Some(mut p) = ctx.data_mut().guest_profiler.take() { + p.sample(&ctx, std::time::Duration::ZERO); + ctx.data_mut().guest_profiler = Some(p); + } + Ok(wasmtime::UpdateDeadline::Continue(1)) + }); + let ticker_engine = engine.clone(); + std::thread::spawn(move || { + loop { + std::thread::sleep(SAMPLE_INTERVAL); + ticker_engine.increment_epoch(); + } + }); + } store.data_mut().stdlib_root = std::env::var("PYRE_STDLIB").ok(); if let Some(n) = fuel_limit { store.set_fuel(n)?; @@ -263,6 +303,14 @@ fn run(module_path: &PathBuf, source: &str) -> Result { if fuel_limit.is_some() { let _ = store.set_fuel(u64::MAX); } + if let Some(path) = &guest_profile_out { + if let Some(p) = store.data_mut().guest_profiler.take() { + let f = std::fs::File::create(path).context("create guest profile output")?; + p.finish(std::io::BufWriter::new(f)) + .context("write guest profile")?; + eprintln!("[guest-profile] wrote {path}"); + } + } if std::env::var_os("PYRE_WASM_JIT_STATS").is_some() { let lin_mem = memory.data_size(&store); // Split linear-memory growth into GC-retained vs. host-heap: a leak that @@ -310,7 +358,7 @@ fn run(module_path: &PathBuf, source: &str) -> Result { } // compile_bridge outcome tallies (diagnostic). 0=entered 1=declCALL_ASM // 2=declMultiPeel 3=declNotDirect 4=declRefHome 5=BRIDGE_OK - // 6=loopClosing 7=srcHasPreamble. + // 6=loopClosing 7=srcHasPreamble 15=declCAHostTrampoline. if let Ok(diag) = instance.get_typed_func::(&mut store, "pyre_jit_bridge_diag") { let labels = [ "entered", @@ -327,8 +375,8 @@ fn run(module_path: &PathBuf, source: &str) -> Result { "decl_noadvance", "ca_cell_set", "ca_cells_zero", - "decl_ca_chain", - "reserved15", + "reserved14", + "decl_ca_trampoline", ]; let mut parts = Vec::new(); for (i, lbl) in labels.iter().enumerate() {