diff --git a/majit/gate-triage.md b/majit/gate-triage.md index c48840ef346..3e9ec3f8957 100644 --- a/majit/gate-triage.md +++ b/majit/gate-triage.md @@ -28,6 +28,7 @@ cover the condition they diagnose. | `MAJIT_GC_FREELIST_DIAG` | OFF | Reports GC freelist allocation and reuse; remove when freelist accounting has sufficient invariant tests. | | `MAJIT_GC_ITEMSBLOCK` | ON | Selects GC-managed list item blocks; `0`, `off`, or `false` restores the fallback, which can be removed after deleting the alternate representation. | | `MAJIT_JTRANSFORM_SHADOW` | OFF | Compares shadow and primary jtransform results; remove after deleting the shadow implementation. | +| `MAJIT_LEFTOVER` | OFF | Prints leftover-empty / leftover_peel_tos compile and runtime dumps; remove when leftover-empty FOR_ITER reload is covered by ordinary tests. | | `MAJIT_MIR_FRAMESTATE` | ON | Selects framestate-threaded MIR lowering; `0` or `false` restores the older lowering, and the escape hatch retires with that path. | | `MAJIT_MIR_FRAMESTATE_DEBUG` | OFF | Prints framestate merge diagnostics; remove when merge failures are covered by focused tests. | | `MAJIT_MIR_FRAMESTATE_STRICT` | OFF | Turns framestate fallback into a hard failure; remove after deleting the fallback. | @@ -331,6 +332,13 @@ cover the condition they diagnose. - What it does: Emits the resume tag, value, and null status of the virtualizable identity slot consumed by `consume_vable_info()`. - Retirement condition: Remove after the unseeded-snapshot route into `_number_boxes()` is rejected or proven unreachable. +### `MAJIT_LEFTOVER` + +- Read sites: 3 — `majit/majit-metainterp/src/compile.rs`, `majit/majit-metainterp/src/pyjitpl.rs`, `pyre/pyre-jit/src/eval.rs` +- Accessor: `std::env::var_os("MAJIT_LEFTOVER")` at leftover-empty compile dumps, leftover_peel_tos, and `loop_red_frame` +- What it does: Prints leftover-empty field-walk / leftover_peel_tos compile and runtime dumps so a FOR_ITER leftover remapped onto a non-iterator portal TOS (ZipInfo) can be distinguished from a live listiter reload. +- Retirement condition: Remove when leftover-empty FOR_ITER reload is covered by ordinary tests (`compile::tests` leftover_peel_tos / reject) and ensurepip no longer needs the dump. + ### `MAJIT_LLBC_EXTRACTION` - Read sites: 2 — `pyre/pyre-jit-trace/build.rs` diff --git a/majit/gc-root-brackets.baseline.json b/majit/gc-root-brackets.baseline.json index 22454748968..bde9a6253c3 100644 --- a/majit/gc-root-brackets.baseline.json +++ b/majit/gc-root-brackets.baseline.json @@ -14,6 +14,7 @@ "unbracketed_calls_fns": 826, "unmatched_seeds": [ "majit_gc::standalone_alloc_fast_nursery_collecting_typed_rooted", + "majit_gc::standalone_alloc_fast_nursery_collecting_typed_roots", "majit_gc::standalone_alloc_nursery_collecting_typed_rooted" ] }, @@ -32,6 +33,7 @@ "unbracketed_calls_fns": 836, "unmatched_seeds": [ "majit_gc::standalone_alloc_fast_nursery_collecting_typed_rooted", + "majit_gc::standalone_alloc_fast_nursery_collecting_typed_roots", "majit_gc::standalone_alloc_nursery_collecting_typed_rooted" ] } diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index 3b2a9a937cf..24be3cbde23 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -377,6 +377,9 @@ fn register_active_hooks(supports_guard_gc_type: bool) { majit_gc::set_active_alloc_nursery_collecting_typed_rooted(Some( alloc_nursery_collecting_typed_rooted_via_active_runtime, )); + majit_gc::set_active_alloc_nursery_collecting_typed_roots(Some( + alloc_nursery_collecting_typed_roots_via_active_runtime, + )); majit_gc::set_active_alloc_oldgen_typed(Some(alloc_oldgen_typed_via_active_runtime)); majit_gc::set_active_collect_generation(Some(collect_generation_via_active_runtime)); majit_gc::set_active_collect_step(Some(collect_step_via_active_runtime)); @@ -1751,6 +1754,32 @@ unsafe fn alloc_nursery_collecting_typed_rooted_via_active_runtime( .unwrap_or(GcRef(0)) } +/// Rooted companion used when more than one native Rust slot holds a GC child. +/// MiniMark registers those slots only on the nursery-full slow path. +/// +/// # Safety +/// `roots` must address `root_count` contiguous mutable [`GcRef`] slots +/// which remain valid until this call returns. +/// `needs_write_barrier` must remain a valid mutable `bool` slot. +unsafe fn alloc_nursery_collecting_typed_roots_via_active_runtime( + type_id: u32, + size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, +) -> GcRef { + with_cranelift_gc(|gc| unsafe { + gc.alloc_fast_nursery_collecting_typed_roots( + type_id, + size, + roots, + root_count, + needs_write_barrier, + ) + }) + .unwrap_or(GcRef(0)) +} + /// `majit_gc::AllocOldgenTypedFn` installed by `set_gc_allocator`. /// Routes host-side allocations that need a stable (non-moving) /// pointer through the active cranelift-owned GC's old-gen. Used by @@ -2143,7 +2172,7 @@ static CALL_ASSEMBLER_FORCE_FN: OnceLock i64> = OnceLock:: /// `compile.py:710-716 resume_in_blackhole(descr, deadframe)` parity: /// callback to resume execution from the guard failure point using the /// blackhole interpreter. Args: `(descr_addr, rebuilt_values_ptr, -/// num_rebuilt, raw_deadframe_ptr, num_raw, guard_exc)` → +/// num_rebuilt, raw_deadframe_ptr, num_raw, guard_exc, savedata)` → /// `Option`. The receiver recovers the failed descr from /// `descr_addr` via `Backend::fail_descr_arc_from_addr` /// (`history.py:125` `cpu.get_latest_descr` parity) and derives @@ -2155,14 +2184,20 @@ static CALL_ASSEMBLER_FORCE_FN: OnceLock i64> = OnceLock:: /// stub staged into `jf_guard_exc`, handed to the blackhole resume per /// `blackhole.py _prepare_resume_from_failure`. `0` = no pending /// exception. +/// +/// `savedata` is `cpu.get_savedata_ref(deadframe)` (`llmodel.py`): the +/// `jf_savedata` AllVirtuals cache a `GUARD_NOT_FORCED` force already +/// materialized. `0` = no cache. type CallAssemblerBlackholeFn = fn(usize, *mut majit_backend::jitframe::JitFrame, i64) -> Option; static CALL_ASSEMBLER_BLACKHOLE_FN: OnceLock = OnceLock::new(); /// Register a blackhole callback for call_assembler guard failure resume. -/// The trailing `i64` is `cpu.grab_exc_value(deadframe)` (llmodel.py): -/// the callee's `jf_guard_exc` slot, forwarded so the blackhole resume can -/// seed `_prepare_resume_from_failure` (blackhole.py). +/// The last two arguments are `cpu.grab_exc_value(deadframe)` (`i64`) and +/// `cpu.get_savedata_ref(deadframe)` (`usize`): the callee's `jf_guard_exc` +/// and `jf_savedata` slots, forwarded so the blackhole resume can seed +/// `_prepare_resume_from_failure` (blackhole.py) and reuse a +/// `GUARD_NOT_FORCED` AllVirtuals cache. pub fn register_call_assembler_blackhole(f: CallAssemblerBlackholeFn) { let _ = CALL_ASSEMBLER_BLACKHOLE_FN.set(f); } @@ -3120,6 +3155,7 @@ pub fn set_savedata_ref_on_deadframe( let jf = frame .as_jitframe_mut() .ok_or_else(|| BackendError::Unsupported("expected JitFrameDeadFrame".to_string()))?; + majit_gc::gc_write_barrier(jf.jf_gcref()); jf.set_savedata_ref(data); Ok(()) } @@ -3196,6 +3232,18 @@ fn grab_exc_value_from_jf_ptr(jf_ptr: usize) -> i64 { unsafe { *((jf_ptr + JF_GUARD_EXC_OFS as usize) as *const usize) as i64 } } +/// `cpu.get_savedata_ref(deadframe)` for the raw JITFRAME handed to the +/// CALL_ASSEMBLER guard helper. Unlike [`grab_exc_value_from_jf_ptr`], this +/// field is not consumed: `ResumeGuardForcedDescr.handle_fail` reveals it +/// during the immediately following blackhole resume. +#[inline] +fn get_savedata_from_jf_ptr(jf_ptr: usize) -> usize { + if jf_ptr == 0 { + return 0; + } + unsafe { *((jf_ptr + JF_SAVEDATA_OFS as usize) as *const usize) } +} + fn execute_registered_loop_target(target: &RegisteredLoopTarget, inputs: &[i64]) -> DeadFrame { let mut cur_code_ptr = target.code_ptr; // Borrowed, not cloned — see `execute_with_inputs_at_dispatch_key`. @@ -4093,6 +4141,17 @@ extern "C" fn gc_alloc_typed_nursery_shim(type_id: u64, size: u64) -> u64 { }) } +/// Leftover `New` / `NewWithVtable` twin of [`gc_alloc_typed_nursery_shim`] +/// for a `non_moving` size descr. Rewrite normally lowers those to +/// `malloc_big_fixedsize_oldgen`; this is the same allocator if a +/// `NewWithVtable` still reaches the backend. +extern "C" fn gc_alloc_typed_oldgen_shim(type_id: u64, size: u64) -> u64 { + oom_signal_if_zero(active_runtime_alloc_oldgen_typed( + type_id as u32, + size as usize, + )) +} + extern "C" fn gc_alloc_varsize_shim( base_size: u64, item_size: u64, @@ -7389,7 +7448,7 @@ fn emit_guard_exit( let mut publish = PairedSlotStores::default(); for (slot, &arg_ref) in info.fail_arg_refs.iter().enumerate() { - let offset = JF_FRAME_ITEM0_OFS + (slot as i32) * 8; + let offset = JF_FRAME_ITEM0_OFS + (info.fail_locs[slot] as i32) * 8; // resume.py failargs may contain None holes. Keep the slot numbering // positional, while leaving the dead slot unwritten like PyPy's @@ -8566,6 +8625,11 @@ struct GuardInfo { fail_index: u32, can_have_bridge: bool, fail_arg_refs: Vec, + /// assembler.py `store_info_on_descr`: physical frame locations, in + /// logical fail-argument order. GUARD_NOT_FORCED_2 keeps slot zero free + /// for `genop_finish`'s return value, like a spilled FrameLoc rather than + /// the return register's save slot. + fail_locs: Vec, /// The GUARD_VALUE operand this exit stores in the trace's counter slot /// so `make_a_counter_per_value` has a slot to name, paired with that /// slot. See `counter_value_spill`. @@ -9825,10 +9889,20 @@ impl CraneliftBackend { .filter(|&&slot| slot != 0xFFFF) .map(|&slot| slot as usize) .collect(); + // Stores write `force_spill_base + fail-arg index`. + // `collect_guards` recorded Ref slots at + // `fail_loc_base + index` (`fail_loc_base == 1` for + // GUARD_NOT_FORCED_2) using `resolve_fail_arg_types`. + // Re-base those slots; do not re-filter with + // `OpRef::ty()`, which is the redefining op's variant + // and can drop a live Ref. info.failarg_ref_slots = info .failarg_ref_slots .iter() - .map(|slot| force_spill_base + slot) + .map(|slot| force_spill_base + (slot - 1)) + .collect(); + info.fail_locs = (0..info.fail_arg_refs.len()) + .map(|index| force_spill_base + index) .collect(); info.gcmap = allocate_gcmap(&info.failarg_ref_slots); // opassembler.py `_finish_gcmap`: the map armed by @@ -15559,9 +15633,15 @@ impl CraneliftBackend { OpCode::New | OpCode::NewWithVtable => { let __descr_arc_sd = op.getdescr(); let sd = __descr_arc_sd.as_ref().and_then(|d| d.as_size_descr()); - let (size, type_id, vtable) = sd.map_or((16, 0, 0usize), |sd| { - (sd.size() as i64, sd.type_id() as i64, sd.vtable()) - }); + let (size, type_id, vtable, non_moving) = + sd.map_or((16, 0, 0usize, false), |sd| { + ( + sd.size() as i64, + sd.type_id() as i64, + sd.vtable(), + sd.non_moving(), + ) + }); let size_val = builder.ins().iconst(cl_types::I64, size); let type_id_val = builder.ins().iconst(cl_types::I64, type_id); // llmodel.py bh_new_with_vtable: @@ -15578,6 +15658,15 @@ impl CraneliftBackend { && vtable_offset.is_some(); let vtable_off_i32 = vtable_offset.unwrap_or(0) as i32; if cranelift_gc_active() { + // `rewrite.rs handle_new`: a `non_moving` descr declines + // the nursery and allocates through the old-generation + // twin. Honor the same flag if NewWithVtable still + // reaches the backend (wasm leftover New does this). + let alloc_shim = if non_moving { + gc_alloc_typed_oldgen_shim as *const () as usize + } else { + gc_alloc_typed_nursery_shim as *const () as usize + }; let cur_jf = builder.ins().get_pinned_reg(ptr_type); let result = emit_collecting_gc_call( &mut builder, @@ -15590,7 +15679,7 @@ impl CraneliftBackend { &demoted_failarg_slots, ref_root_base_ofs, per_call_gcmap, - gc_alloc_typed_nursery_shim as *const () as usize, + alloc_shim, &[type_id_val, size_val], Some(cl_types::I64), ) @@ -16322,6 +16411,7 @@ fn precompute_max_output_slots(inputargs: &[InputArg], ops: &[Op]) -> usize { } else { num_inputs }; + let n = n + usize::from(op.opcode == OpCode::GuardNotForced2); if n > max_slots { max_slots = n; } @@ -16351,6 +16441,10 @@ fn collect_guards( let type_index = OpTypeIndex::new(inputargs, ops); let (type_overrides, op_def_positions) = build_type_overrides(ops, &type_index); + // assembler.py `store_force_descr` / `genop_finish`: the terminal + // force guard's reference spills remain roots after FINISH. + let mut finish_gcmap_slots = Vec::new(); + // Map Label descr index → block arity, used to distinguish internal vs // external JUMPs. rewriter.py LABEL/JUMP redirect parity: a JUMP whose // descr targets a Label in this function but with a *different* arg arity @@ -16459,7 +16553,11 @@ fn collect_guards( let counter_value_spill = counter_value_spill(op, &fail_arg_refs) .zip(counter_slot) .inspect(|&(_, slot)| *max_output_slots = (*max_output_slots).max(slot + 1)); - let n = fail_arg_refs.len(); + let fail_loc_base = usize::from(op.opcode == OpCode::GuardNotForced2); + let fail_locs: Vec = (0..fail_arg_refs.len()) + .map(|index| fail_loc_base + index) + .collect(); + let n = fail_arg_refs.len() + fail_loc_base; if n > *max_output_slots { *max_output_slots = n; } @@ -16858,7 +16956,7 @@ fn collect_guards( is_finish || is_external_jump || !arg_ref.is_constant(), "regalloc.py:1206: guard fail_args must not contain Const (slot={i}, opref={arg_ref:?})" ); - slots.push(i); + slots.push(fail_locs[i]); } } slots @@ -17110,6 +17208,29 @@ fn collect_guards( } else { None }; + if op.opcode == OpCode::GuardNotForced2 { + // llsupport/assembler.py `store_info_on_descr`: force() reads + // exactly the locations written by the register allocator. + as_fd(&descr).set_rd_locs( + fail_arg_refs + .iter() + .zip(&fail_locs) + .map(|(arg, &loc)| { + if arg.is_none() { + 0xFFFF + } else { + u16::try_from(loc) + .expect("force failarg frame location exceeds rd_locs") + } + }) + .collect(), + ); + finish_gcmap_slots.clone_from(&failarg_ref_slots); + } + let mut gcmap_slots = failarg_ref_slots.clone(); + if is_finish { + gcmap_slots.append(&mut finish_gcmap_slots); + } fail_descrs.push(descr); fail_descr_cells.push(cell); // assembler.py must_save_exception parity: @@ -17140,6 +17261,7 @@ fn collect_guards( fail_index, can_have_bridge, fail_arg_refs, + fail_locs, counter_value_spill, must_save_exception, // llsupport/assembler.py `GuardToken.compute_gcmap` walks @@ -17149,7 +17271,7 @@ fn collect_guards( // ports the same split (`guard_gcmap_from_faillocs`). Anything a // guard exit does not itself write must stay out of this map. bridge_source_slots, - gcmap: allocate_gcmap(&failarg_ref_slots), + gcmap: allocate_gcmap(&gcmap_slots), failarg_ref_slots, fail_descr_ptr, bridge_cache_addrs, @@ -26920,33 +27042,33 @@ mod tests { #[test] fn test_deadframe_drop_preserves_the_frames_gcmap() { let mut gc = MiniMarkGC::with_config(GcConfig { - nursery_size: 160, - large_object_threshold: 1024, + nursery_size: 1 << 20, + large_object_threshold: 1 << 20, ..GcConfig::default() }); gc.register_type(TypeInfo::simple(16)); let root = gc.alloc_with_type(0, 16); + unsafe { + *(root.0 as *mut u64) = 0xF012_CED; + } let mut backend = backend_with_gc(gc); let inputargs = vec![InputArg::new_ref(0)]; + let guard = mk_op(OpCode::GuardNotForced2, &[], OpRef::NONE.raw()); + guard.setfailargs(smallvec::smallvec![rb(OpRef::input_arg_ref(0))]); let ops = vec![ - mk_op(OpCode::Label, &[OpRef::input_arg_ref(0)], OpRef::NONE.raw()), - mk_op( - OpCode::Finish, - &[OpRef::input_arg_ref(0)], - OpRef::NONE.raw(), - ), + mk_op(OpCode::ForceToken, &[], 1), + guard, + mk_op(OpCode::Finish, &[OpRef::ref_op(1)], OpRef::NONE.raw()), ]; let token = JitCellToken::new(1509); backend.compile_loop(&inputargs, &ops, &token).unwrap(); let frame = backend.execute_token(&token, &[Value::Ref(root)]); - let jf = frame - .as_jitframe() - .expect("cranelift deadframes are JitFrameDeadFrame") - .jf_gcref(); + let jf = backend.get_ref_value(&frame, 0); + let escaped = majit_gc::shadow_stack::OwnerRootGuard::new(jf); drop(frame); // Nothing allocates between the release and this read, so the frame is // still where the root last named it. @@ -27140,6 +27262,34 @@ mod tests { // Callee has ForceToken + GuardNotForced2 + Finish(force_token). // Caller uses CallAssemblerR and gets the force_token result. + #[test] + fn test_guard_not_forced_2_keeps_failargs_after_finish() { + // runner_test.py `test_guard_not_forced_2`: force a returned token, + // not just a token inside a still-running CALL_MAY_FORCE. FINISH's + // result slot must not overwrite the guard's frame locations. + let mut backend = CraneliftBackend::new(); + let inputargs = vec![InputArg::new_int(0), InputArg::new_int(1)]; + let guard = mk_op(OpCode::GuardNotForced2, &[], OpRef::NONE.raw()); + guard.setfailargs(smallvec::smallvec![rb(OpRef::int_op(2))]); + let ops = vec![ + mk_op( + OpCode::IntAdd, + &[OpRef::input_arg_int(0), OpRef::input_arg_int(1)], + 2, + ), + mk_op(OpCode::ForceToken, &[], 3), + guard, + mk_op(OpCode::Finish, &[OpRef::ref_op(3)], OpRef::NONE.raw()), + ]; + let token = JitCellToken::new(9017); + backend.compile_loop(&inputargs, &ops, &token).unwrap(); + let frame = backend.execute_token(&token, &[Value::Int(20), Value::Int(10)]); + let force_token = backend.get_ref_value(&frame, 0); + assert!(!force_token.is_null()); + let forced = force_token_to_dead_frame(force_token); + assert_eq!(get_int_from_deadframe(&forced, 0).unwrap(), 30); + } + #[test] fn test_all_guards_have_recovery_layout() { let mut backend = CraneliftBackend::new(); diff --git a/majit/majit-backend-dynasm/src/aarch64/assembler.rs b/majit/majit-backend-dynasm/src/aarch64/assembler.rs index bac9cfbea8e..5e6c9679e65 100644 --- a/majit/majit-backend-dynasm/src/aarch64/assembler.rs +++ b/majit/majit-backend-dynasm/src/aarch64/assembler.rs @@ -6226,11 +6226,6 @@ impl<'a> AssemblerARM64<'a> { .copied() .expect("call_assembler missing rewritten jitframe arg"); let vable_loc = arglocs.get(1).copied(); - // aarch64/regalloc.py:661-664 routes CALL_ASSEMBLER through - // `_call(..., gc_level=2)`, which spills all managed registers. - // x19 is already saved by the JIT prologue, so use it as scratch - // here without an extra call-site stack save. - dynasm!(self.mc ; .arch aarch64 ; mov x19, x29); self.emit_load_to_rax(frame_loc); let descr_arc = op.getdescr(); @@ -6261,9 +6256,6 @@ impl<'a> AssemblerARM64<'a> { if !is_resolved { let force_addr = crate::call_assembler_force_fn_addr() as i64; - dynasm!(self.mc ; .arch aarch64 - ; mov x29, x19 - ); if force_addr != 0 { if let Some(vloc) = vable_loc { self.emit_load_to_rax(vloc); @@ -6308,9 +6300,13 @@ impl<'a> AssemblerARM64<'a> { } else if let Some(entry_label) = self.self_entry_label { dynasm!(self.mc ; .arch aarch64 ; bl =>entry_label); } - dynasm!(self.mc ; .arch aarch64 - ; mov x29, x19 - ); + // aarch64/callbuilder.py `CallBuilderARM64.pop_gcmap` calls + // `AssemblerARM64._reload_frame_if_necessary` before clearing + // the map. The callee footer restores the pre-call x29 from the + // C stack, but that address may be stale after a minor collection + // moved the caller jitframe. Reload x29 from the shadow stack; + // keeping the old address in an allocatable callee-saved register + // is both non-orthodox and unsound. self.pop_pending_call_gcmap_after_collect(pushed_gcmap); let fast_path = self.mc.new_dynamic_label(); diff --git a/majit/majit-backend-dynasm/src/regalloc.rs b/majit/majit-backend-dynasm/src/regalloc.rs index 22d632bd934..0f9d5f6a29d 100644 --- a/majit/majit-backend-dynasm/src/regalloc.rs +++ b/majit/majit-backend-dynasm/src/regalloc.rs @@ -2888,8 +2888,11 @@ impl<'a> RegAlloc<'a> { | GuardKind::Overflow | GuardKind::NotInvalidated | GuardKind::FutureCondition - | GuardKind::NotForced | GuardKind::AlwaysFails => self.consider_guard_no_args_j2(fail_args, i, output), + GuardKind::NotForced if op.opcode == OpCode::GuardNotForced2 => { + self.consider_guard_not_forced_2_j2(fail_args, i, output) + } + GuardKind::NotForced => self.consider_guard_no_args_j2(fail_args, i, output), GuardKind::Exception => { self.consider_guard_exception_j2(args, fail_args, op, i, output) } @@ -5375,29 +5378,31 @@ impl<'a> RegAlloc<'a> { } } - /// llsupport/regalloc.py locs_for_call_assembler parity. - /// RPython syncs args to stack, then before_call spills everything. - /// We force-sync register args to frame first via _sync_var_to_stack, - /// then before_call spills remaining. arglocs after before_call are - /// all Frame or Immed — no register-clobber issues during calloc. + /// llsupport/regalloc.py `locs_for_call_assembler` parity. + /// + /// RPython syncs only argument 1 (the virtualizable) to the frame. The + /// callee jitframe in argument 0 stays in its current location, captured + /// before `before_call`. fn consider_call_assembler(&mut self, op: &Op, i: usize, output: &mut Vec) { - // llsupport/regalloc.py: self.rm._sync_var_to_stack(op.getarg(k)) - // Force all register-held args to frame before before_call. - for arg in op.getarglist().iter() { - if arg.is_constant() { - continue; - } - let arg = arg.to_opref(); - let tp = self.tp(arg); + assert!(matches!(op.num_args(), 1 | 2)); + if op.num_args() == 2 { + let vable = op.arg(1).to_opref(); + let tp = self.tp(vable); if tp == Type::Float { self.xrm - ._sync_var_to_stack(arg, tp, &mut self.longevity, &mut self.fm); + ._sync_var_to_stack(vable, tp, &mut self.longevity, &mut self.fm); } else { self.rm - ._sync_var_to_stack(arg, tp, &mut self.longevity, &mut self.fm); + ._sync_var_to_stack(vable, tp, &mut self.longevity, &mut self.fm); } } + let mut arglocs = Vec::with_capacity(op.num_args()); + for arg in op.getarglist().iter() { + let arg = arg.to_opref(); + arglocs.push(self.loc(arg, self.tp(arg))); + } + let type_index = OpTypeIndex::from_parts( self.inputargs, self.operations, @@ -5421,14 +5426,6 @@ impl<'a> RegAlloc<'a> { &type_index, ); - // After before_call, all args are in Frame or Const — safe for calloc. - let mut arglocs: Vec = Vec::new(); - for arg in op.getarglist().iter() { - let arg = arg.to_opref(); - let tp = self.tp(arg); - arglocs.push(self.loc_must_exist(arg, tp)); - } - let result_tp = op.opcode.result_type(); let result_loc = if result_tp != Type::Void { let r = if result_tp == Type::Float { @@ -5452,20 +5449,24 @@ impl<'a> RegAlloc<'a> { i: usize, output: &mut Vec, ) { - for &arg in args { - if arg.is_constant() { - continue; - } - let tp = self.tp(arg); + assert!(matches!(args.len(), 1 | 2)); + if args.len() == 2 { + let vable = args[1]; + let tp = self.tp(vable); if tp == Type::Float { self.xrm - ._sync_var_to_stack(arg, tp, &mut self.longevity, &mut self.fm); + ._sync_var_to_stack(vable, tp, &mut self.longevity, &mut self.fm); } else { self.rm - ._sync_var_to_stack(arg, tp, &mut self.longevity, &mut self.fm); + ._sync_var_to_stack(vable, tp, &mut self.longevity, &mut self.fm); } } + let mut arglocs = Vec::with_capacity(args.len()); + for &arg in args { + arglocs.push(self.loc(arg, self.tp(arg))); + } + let type_index = OpTypeIndex::from_parts( self.inputargs, self.operations, @@ -5489,12 +5490,6 @@ impl<'a> RegAlloc<'a> { &type_index, ); - let mut arglocs: Vec = Vec::new(); - for &arg in args { - let tp = self.tp(arg); - arglocs.push(self.loc_must_exist(arg, tp)); - } - let result_tp = op.opcode.result_type(); let result_loc = if result_tp != Type::Void { let dst = dst.unwrap_or(op.pos().get()); diff --git a/majit/majit-backend-dynasm/src/runner.rs b/majit/majit-backend-dynasm/src/runner.rs index 290cbf3f5fe..cb089395dab 100644 --- a/majit/majit-backend-dynasm/src/runner.rs +++ b/majit/majit-backend-dynasm/src/runner.rs @@ -386,6 +386,9 @@ fn register_active_hooks(supports_guard_gc_type: bool) { majit_gc::set_active_alloc_nursery_collecting_typed_rooted(Some( dynasm_alloc_nursery_collecting_typed_rooted, )); + majit_gc::set_active_alloc_nursery_collecting_typed_roots(Some( + dynasm_alloc_nursery_collecting_typed_roots, + )); majit_gc::set_active_alloc_oldgen_typed(Some(dynasm_alloc_oldgen_typed)); majit_gc::set_active_collect_generation(Some(dynasm_collect_generation)); majit_gc::set_active_collect_step(Some(dynasm_collect_step)); @@ -706,6 +709,35 @@ unsafe fn dynasm_alloc_nursery_collecting_typed_rooted( } } +unsafe fn dynasm_alloc_nursery_collecting_typed_roots( + type_id: u32, + size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, +) -> GcRef { + if let Some(r) = gc_box::with_mut(|g| unsafe { + g.alloc_fast_nursery_collecting_typed_roots( + type_id, + size, + roots, + root_count, + needs_write_barrier, + ) + }) { + return r; + } + unsafe { + majit_gc::standalone_alloc_fast_nursery_collecting_typed_roots( + type_id, + size, + roots, + root_count, + needs_write_barrier, + ) + } +} + /// Host-side old-gen allocation trampoline. Used by /// pyre-object allocators (`w_int_new`, `w_float_new`) whose /// callers cannot register the returned pointer as a GC root before @@ -3216,6 +3248,7 @@ impl Backend for DynasmBackend { // grab_exc_value (llmodel.py): read jf_guard_exc off the deadframe // tip before the libc jitframe chain is freed (same as execute_token). let exception_value = GcRef(unsafe { (*result_jf).jf_guard_exc }); + let savedata = GcRef(unsafe { (*result_jf).jf_savedata }); let guard_value_operand = majit_backend::guard_value_counter_slot(descr_fd) .map(|slot| unsafe { crate::llmodel::get_int_value_direct(result_jf, slot) as i64 }); @@ -3230,7 +3263,7 @@ impl Backend for DynasmBackend { outputs, typed_outputs, exit_layout, - savedata: None, + savedata: (!savedata.is_null()).then_some(savedata), exception_value, fail_index: descr_fd.fail_index_per_trace(), trace_id: descr_fd.trace_id(), @@ -3314,6 +3347,27 @@ impl Backend for DynasmBackend { } } + fn set_savedata_ref(&self, frame: &mut DeadFrame, data: GcRef) { + match frame { + DeadFrame::JitFrame(jf) => { + // llmodel.py set_savedata_ref is a GCREF field store. + majit_gc::gc_write_barrier(jf.jf_gcref()); + jf.set_savedata_ref(data); + } + DeadFrame::LibcJitFrame(jf) => jf.set_savedata_ref(data), + DeadFrame::Boxed(_) => panic!("dynasm deadframe is a jitframe"), + } + } + + fn get_savedata_ref(&self, frame: &DeadFrame) -> Option { + let data = match frame { + DeadFrame::JitFrame(jf) => jf.get_savedata_ref(), + DeadFrame::LibcJitFrame(jf) => jf.get_savedata_ref(), + DeadFrame::Boxed(_) => panic!("dynasm deadframe is a jitframe"), + }; + (!data.is_null()).then_some(data) + } + fn clear_stored_exception(&self) { crate::jit_exc_clear(); } diff --git a/majit/majit-backend-wasm/src/codegen.rs b/majit/majit-backend-wasm/src/codegen.rs index bc0e7f8d3bd..dd48db863dd 100644 --- a/majit/majit-backend-wasm/src/codegen.rs +++ b/majit/majit-backend-wasm/src/codegen.rs @@ -2412,11 +2412,11 @@ impl HomeLiveness { Self { def_pos, last_use } } - /// Value `raw` is defined before op `at` and read after it — i.e. its - /// local holds a value a collection at op `at` could invalidate. + /// Value `raw` is defined before op `at` and still read at or after it. + /// RPython `RegisterManager.is_still_alive` is `last_usage >= position`. fn live_across(&self, raw: u32, at: usize) -> bool { let raw = raw as usize; - raw < self.def_pos.len() && self.def_pos[raw] < at as i32 && self.last_use[raw] > at as i32 + raw < self.def_pos.len() && self.def_pos[raw] < at as i32 && self.last_use[raw] >= at as i32 } fn live_across_any(&self, raw: u32, positions: &[usize]) -> bool { @@ -2579,6 +2579,7 @@ fn emit_reload_frame_if_necessary( residual_type_base: Option, ca_reload_fn_ptr: i64, jf_top_addr: Option, + wb: &WriteBarrierHelpers, ) { if let Some(top_addr) = jf_top_addr { // assembler.py:1369-1377: reload the possibly-forwarded top JitFrame @@ -2598,6 +2599,42 @@ fn emit_reload_frame_if_necessary( // — it published no reload helper, and reloading from a shadow stack // that never held this frame would install an unrelated one. } + if jf_top_addr.is_some() || ca_reload_fn_ptr != 0 { + emit_frame_write_barrier(sink, residual_type_base, wb); + } +} + +/// x86/assembler.py `_reload_frame_if_necessary` reapplies the non-array +/// barrier after reloading the frame. A minor collection can promote it; +/// subsequent Ref spills must then keep it in the remembered set even after +/// it leaves the shadow stack. The result of the collecting call may still +/// be on the wasm operand stack, so this fast path is stack-neutral. +fn emit_frame_write_barrier( + sink: &mut PeepSink<'_, '_>, + residual_type_base: Option, + wb: &WriteBarrierHelpers, +) { + if wb.fn_ptr == 0 { + return; + } + let base = residual_type_base.expect("frame barrier needs the one-argument helper type"); + sink.local_get(0); + sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); + sink.i32_sub(); + sink.i32_const(wb.flag_byteofs); + sink.i32_add(); + sink.i32_load8_u(memarg(0, 0)); + sink.i32_const(wb.if_flag as i32); + sink.i32_and(); + sink.if_(BlockType::Empty); + sink.local_get(0); + sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); + sink.i32_sub(); + sink.i64_extend_i32_u(); + sink.i32_const(wb.fn_ptr as i32); + sink.call_indirect(0, base + 1); + sink.drop(); + sink.end(); } /// CA-arm-only variant of [`emit_reload_frame_if_necessary`]. The direct CA @@ -2608,13 +2645,15 @@ fn emit_reload_ca_frame_if_necessary( residual_type_base: Option, ca_reload_fn_ptr: i64, ca_inline: Option, + wb: &WriteBarrierHelpers, ) { if let Some(inline) = ca_inline { debug_assert!(residual_type_base.is_some()); emit_ca_reload_top(sink, inline.jf_top_addr); sink.local_set(0); + emit_frame_write_barrier(sink, residual_type_base, wb); } else { - emit_reload_frame_if_necessary(sink, residual_type_base, ca_reload_fn_ptr, None); + emit_reload_frame_if_necessary(sink, residual_type_base, ca_reload_fn_ptr, None, wb); } } @@ -4224,10 +4263,11 @@ fn new_inline_nursery_member( na: &NurseryAllocParams, constants: &indexmap::IndexMap, ) -> Option<(usize, u32)> { - let result_id = op.pos().get().raw(); - if OpRef::raw_is_constant(result_id) { + let result = op.pos().get(); + if result.is_constant() { return None; } + let result_id = result.raw(); match op.opcode { OpCode::New | OpCode::NewWithVtable => { let descr = op.getdescr()?; @@ -4406,7 +4446,166 @@ pub struct AllocHelpers { pub fmod_fn_ptr: i64, } -type BuildWasmModuleOutput = (Vec, Vec, usize, usize); +pub struct WasmModuleData { + pub num_ref_homes: usize, + pub used_label_homes: usize, + /// gcmap.py `allocate_gcmap`: assembler data blocks live with their code. + pub gc_maps: Vec>, + /// Created before code emission so jf_force_descr can name the same + /// stable descriptor the backend publishes for this exit. + pub fail_descrs: Vec>, +} + +/// x86/regalloc.py consider_guard_not_forced_2: keep forced state outside +/// FINISH's return slot. Ref homes are already precise, persistent spills; +/// other arguments use exit slots starting after the return slot. +fn force_arg_location(frame: FrameGeometry, homes: &RefHomes, arg: OpRef, index: usize) -> usize { + let offset = homes + .home(arg) + .map_or(FRAME_SLOT_BASE + (index as u64 + 1) * SLOT_SIZE, |home| { + frame.home_slot_base + home as u64 * SLOT_SIZE + }); + offset as usize / std::mem::size_of::() +} + +type BuildWasmModuleOutput = (Vec, Vec, WasmModuleData); + +/// regalloc.py `get_gcmap` and assembler.py `_finish_gcmap`. Homes are spill +/// locations, not a permanent root set: only live Ref boxes belong in the map +/// at a collecting call. Tracing all homes after FINISH keeps returned child +/// PyFrames and their force-token JITFRAMEs alive recursively. +struct FrameGcMaps { + enabled: bool, + frame: FrameGeometry, + maps: std::cell::RefCell>>, + finish_gcmap: std::cell::RefCell>, +} + +impl FrameGcMaps { + fn new(enabled: bool, frame: FrameGeometry) -> Self { + Self { + enabled, + frame, + maps: Default::default(), + finish_gcmap: Default::default(), + } + } + + fn home_index(&self, home: u32) -> usize { + (self.frame.home_slot_base as usize + home as usize * 8) / std::mem::size_of::() + } + + fn live_indices( + &self, + homes: &RefHomes, + live: &HomeLiveness, + at: usize, + op: &Op, + ) -> Vec { + let mut indices: Vec<_> = homes + .iter() + .filter_map(|(raw, home)| { + // `live_across` is `last_use >= at`, matching RPython + // `RegisterManager.is_still_alive`. `consider_call` still + // force-stores every Ref argument (callbuilder.py); keep + // the same rule here so a collecting Call* whose last + // SSA use is this op stays in the gcmap even if a later + // edit narrows `live_across`. CALL_ASSEMBLER is the same + // rule, not a special case. + let call_arg = op.getarglist().iter().any(|arg| { + let arg = arg.to_opref(); + !arg.is_constant() && arg.raw() == raw + }); + (live.live_across(raw, at) || call_arg).then_some(self.home_index(home)) + }) + .collect(); + // LABEL captures are frozen spill locations shared with chained + // bridges; their values remain live until the source loop resumes. + for home in self.frame.ordinary_home_slots()..self.frame.home_slots { + indices.push(self.home_index(home as u32)); + } + indices + } + + fn force_indices(&self, homes: &RefHomes, op: &Op) -> Vec { + let args = exit_fail_args(op); + let mask = live_fail_arg_mask(op.getdescr().as_ref(), args.len()); + let mut indices: Vec<_> = args + .into_iter() + .zip(mask) + .enumerate() + .filter_map(|(i, (arg, live))| { + // Only a traced home is a GC root. A constant Ref has + // no home; `emit_force_arm` writes it as a literal in + // the force slot, and `force_arg_location` would + // otherwise mark the unrelated positional exit slot. + (live && arg.ty() == Some(Type::Ref) && homes.home(arg).is_some()) + .then(|| force_arg_location(self.frame, homes, arg, i)) + }) + .collect(); + // LABEL captures stay live until the source loop resumes + // (`FrameGeometry`: the whole home region remains covered by + // jf_gcmap). GUARD_NOT_FORCED_2 installs this map until FINISH; + // dropping the captures lets a minor collection free them while + // the old-gen JitFrame stays remembered, and the next + // `live_indices` remarks the leftover. + for home in self.frame.ordinary_home_slots()..self.frame.home_slots { + indices.push(self.home_index(home as u32)); + } + indices + } + + fn emit_push_gcmap(&self, sink: &mut PeepSink<'_, '_>, indices: &[usize]) { + if !self.enabled { + return; + } + // gcmap.py `allocate_gcmap`: length word, then a zeroed bitset. + let word = std::mem::size_of::(); + let bits = word * 8; + let words = self.frame.frame_bytes as usize / word / bits + 1; + let mut map = vec![0usize; words + 1].into_boxed_slice(); + map[0] = words; + for &index in indices { + map[1 + index / bits] |= 1usize << (index % bits); + } + let address = map.as_ptr() as usize; + self.maps.borrow_mut().push(map); + sink.local_get(0); + sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); + sink.i32_sub(); + sink.i32_const(address as i32); + sink.i32_store(mem32(majit_backend::jitframe::JF_GCMAP_OFS as u64)); + } + + fn emit_exit_gcmap(&self, sink: &mut PeepSink<'_, '_>, op: &Op, counter_slot: Option) { + let args = exit_fail_args(op); + let mask = live_fail_arg_mask(op.getdescr().as_ref(), args.len()); + let word = std::mem::size_of::(); + let mut indices: Vec<_> = args + .iter() + .zip(mask) + .enumerate() + .filter_map(|(i, (arg, live))| { + (live && arg.ty() == Some(Type::Ref)) + .then_some((FRAME_SLOT_BASE as usize + i * 8) / word) + }) + .collect(); + if counter_value_spill(op, &args).is_some_and(|arg| arg.ty() == Some(Type::Ref)) { + if let Some(slot) = counter_slot { + indices.push((FRAME_SLOT_BASE as usize + slot as usize * 8) / word); + } + } + if op.opcode == OpCode::Finish { + // assembler.py `genop_finish`: preserve only the forced guard's + // map, plus the Ref return slot. Ordinary temporary homes die. + // Copy, do not drain: one module can emit more than one FINISH + // (merged inline regions), and the second must keep the same + // GUARD_NOT_FORCED_2 homes. + indices.extend(self.finish_gcmap.borrow().iter().copied()); + } + self.emit_push_gcmap(sink, &indices); + } +} /// Counts entries into an out-of-line bridge module and calls out once there /// have been enough of them to pay for merging that bridge into its owner. @@ -5157,10 +5356,11 @@ pub fn build_wasm_module( // this same `(i64×n)->i64` family; make sure arity 2 is declared, // which declares the full 0..=2 range including reload's arity 0. Some(scanned.map_or(2, |m| m.max(2))) - } else if ca.ca_reload_fn_ptr != 0 { + } else if ca.ca_reload_fn_ptr != 0 || ca.jf_top_addr.is_some() { // Every trace body can reload its own frame after a collecting // direct call, even though only bridges emit the CA arm. - Some(scanned.map_or(0, |m| m)) + // The reloaded frame's write barrier takes one argument. + Some(scanned.map_or(1, |m| m.max(1))) } else { scanned } @@ -5446,6 +5646,44 @@ pub fn build_wasm_module( .enumerate() .map(|(i, &arity)| (arity, first_spill_func_idx + i as u32)) .collect(); + let gc_maps = FrameGcMaps::new(ca.ca_reload_fn_ptr != 0 || ca.jf_top_addr.is_some(), *frame); + let mut fail_descrs = Vec::with_capacity(guards.len()); + for (guard, op) in guards.iter().zip( + analysis_ops + .iter() + .filter(|op| op.opcode.is_guard() || op.opcode == OpCode::Finish), + ) { + let rd_locs = if matches!(op.opcode, OpCode::GuardNotForced | OpCode::GuardNotForced2) { + let mask = live_fail_arg_mask(op.getdescr().as_ref(), guard.fail_arg_refs.len()); + let mut locs = Vec::with_capacity(guard.fail_arg_refs.len()); + for (i, (&arg, live)) in guard.fail_arg_refs.iter().zip(mask).enumerate() { + if !live || arg.is_none() { + locs.push(0xFFFF); + continue; + } + let loc = u16::try_from(force_arg_location(*frame, &ref_homes, arg, i)) + .ok() + .filter(|&loc| loc != 0xFFFF) + .ok_or_else(|| { + BackendError::Unsupported("force failarg location exceeds rd_locs".into()) + })?; + locs.push(loc); + } + Some(locs) + } else { + None + }; + fail_descrs.push(Arc::new(crate::failguard::WasmFailDescr { + fail_index: guard.fail_index, + trace_id: 0, // filled by the backend before publishing the code + fail_arg_types: guard.fail_arg_types.clone(), + rd_locs, + is_finish: guard.is_finish, + force_args_offset: frame.force_slot_base as u32, + force_gcmap_ptr: 0, + meta_descr: guard.meta_descr.clone(), + })); + } let func = build_function( inputargs, &analysis_inputargs, @@ -5487,6 +5725,8 @@ pub fn build_wasm_module( label_param_entry, inline_trip.map(|probe| (probe, inline_trip_type_idx)), &spill_helper_indices, + &gc_maps, + &fail_descrs, )?; if label_param_entry { codes.function(&build_label_param_shim(trace_func_idx + 1)); @@ -5498,7 +5738,16 @@ pub fn build_wasm_module( module.section(&codes); let used_labels = label_resume.ref_slots.max(ca.home_gcmap_min_labels); - Ok((module.finish(), guards, num_ref_homes, used_labels)) + Ok(( + module.finish(), + guards, + WasmModuleData { + num_ref_homes, + used_label_homes: used_labels, + gc_maps: gc_maps.maps.into_inner(), + fail_descrs, + }, + )) } fn build_label_param_shim(wide_func_idx: u32) -> Function { @@ -5651,6 +5900,8 @@ fn build_function( label_param_entry: bool, inline_trip: Option<(InlineTripProbe, u32)>, spill_helper_indices: &indexmap::IndexMap, + gc_maps: &FrameGcMaps, + fail_descrs: &[Arc], ) -> Result { // The CA arm requires residual types (the setup above forces arity >= 2 // whenever it is emitted). Its `jit_call` fallback branches are retained @@ -5869,6 +6120,7 @@ fn build_function( counter_slot: counter_slot(inputargs, ops).map(|slot| slot as u64), spill_helpers: spill_helper_indices, gc_table_slots: &gc_table_slots, + gc_maps: Some(gc_maps), }; let mut locals = Vec::new(); let mut start = 0; @@ -6440,6 +6692,13 @@ fn build_function( } else { None }; + if (op.opcode.is_call() && call_can_collect(op)) + || op.opcode.is_malloc() + || (ca.emit_ca && op.opcode.is_call_assembler()) + { + let indices = gc_maps.live_indices(ref_homes, &liveness, op_idx, op); + gc_maps.emit_push_gcmap(&mut sink, &indices); + } match op.opcode { OpCode::Label => {} @@ -6885,21 +7144,19 @@ fn build_function( } OpCode::GuardNotForced => { // x86/assembler.py genop_guard_guard_not_forced: - // `CMP [rbp + jf_descr], 0`, fail when nonzero. `Backend::force` - // stamps that mark on its way out, so this guard is what turns a - // force that landed inside the preceding call into a deopt: the - // trace must not run on holding virtualized fields the force has - // already written back, and the virtuals `handle_async_forcing` - // materialized are attached for THIS exit's resume to consume. - // The bit sits in the upper half of `frame[0]`, so on - // little-endian wasm32 it is bit 0 of the i32 at frame offset - // 4; masking it leaves the `!= 0` the `if` already applies. - const FORCE_TAKEN_HALF_OFS: u64 = 4; - const _: () = assert!(FORCE_TAKEN_BIT == 1 << 32); + // `CMP [rbp + jf_descr], 0`, fail when nonzero. + // `WasmBackend::force` / a synchronous force stamps + // `FORCE_TAKEN_BIT` in `frame[0]` (this backend's + // `jf_descr` word); `emit_force_arm` also writes the + // exit index there, so the test must be the force bit + // and not a plain nonzero check of the header field. sink.local_get(0); - sink.i32_load(memarg(FORCE_TAKEN_HALF_OFS, 2)); - sink.i32_const(1); - sink.i32_and(); + sink.i64_load(mem64(0)); + sink.i64_const(FORCE_TAKEN_BIT); + sink.i64_and(); + sink.i64_const(32); + sink.i64_shr_u(); + sink.i32_wrap_i64(); emit_guard_if_exit( &mut sink, constants, @@ -6918,6 +7175,9 @@ fn build_function( // to test, it is what `store_token_in_vable` emits before a // FINISH so a force arriving while the virtualizable is still // armed can still rebuild a deadframe. Arm, do not test. + let indices = gc_maps.force_indices(ref_homes, op); + gc_maps.emit_push_gcmap(&mut sink, &indices); + *gc_maps.finish_gcmap.borrow_mut() = indices; emit_force_arm( &mut sink, constants, @@ -6925,7 +7185,7 @@ fn build_function( ref_homes, frame, op, - exit_index(op, guard_idx), + &fail_descrs[(guard_idx - fail_index_base) as usize], None, ); guard_idx += 1; @@ -7815,6 +8075,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -7921,6 +8182,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8269,6 +8531,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); emit_reload_frame_if_necessary( @@ -8276,6 +8540,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8420,6 +8685,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8477,6 +8743,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); if !inlined { let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); @@ -8485,6 +8753,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8542,6 +8811,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8582,6 +8852,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); if !inlined { let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); @@ -8590,6 +8862,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8644,6 +8917,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); emit_reload_frame_if_necessary( @@ -8651,6 +8926,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8703,6 +8979,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8759,6 +9036,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); if !inlined { let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); @@ -8767,6 +9046,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -8795,6 +9075,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); } OpCode::ZeroArray => { @@ -8927,6 +9209,8 @@ fn build_function( ops, op_idx, guard_idx, + fail_index_base, + fail_descrs, ); let vi = op.pos().get().raw(); let descr = op @@ -9019,6 +9303,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); sink.local_get(ca_cfp_local); sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); @@ -9044,6 +9330,7 @@ fn build_function( op, frame, ); + emit_frame_write_barrier(&mut sink, residual_type_base, wb); // dispatch key = 0: run the loop from its entry (preamble), not a // LABEL resume — this is a fresh call. sink.local_get(ca_cfp_local); @@ -9172,11 +9459,17 @@ fn build_function( if let (Some(_base), Some(inline)) = (residual_type_base, ca.inline) { emit_ca_reload_caller(&mut sink, inline.jf_top_addr); sink.local_set(0); + // assembler.py `_reload_frame_if_necessary`: reapply the + // non-array barrier after the post-call reload. The + // recursive CA or deopt helper may have promoted the + // caller; later Ref spills must still remember it. + emit_frame_write_barrier(&mut sink, residual_type_base, wb); } else if let Some(base) = residual_type_base { sink.i32_const(ca.ca_reload_caller_fn_ptr as i32); sink.call_indirect(0, base); sink.i32_wrap_i64(); sink.local_set(0); + emit_frame_write_barrier(&mut sink, residual_type_base, wb); } // The frame ABI carries every scalar result as i64 bits. Ref // and Int use those bits directly; Float crosses the local @@ -9240,12 +9533,15 @@ fn build_function( // local 0 already holds the caller from the post-call reload. // The helper path can collect; that is a property of the // callee snapshot, not of this module's ops. + // assembler.py `_reload_frame_if_necessary` then reapplies + // the non-array write barrier after the reload. if ca.inline.is_none() { emit_reload_ca_frame_if_necessary( &mut sink, residual_type_base, ca.ca_reload_fn_ptr, ca.inline, + wb, ); } else { sink.i32_const(dispatch_entry); @@ -9256,6 +9552,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.inline, + wb, ); sink.end(); } @@ -9312,6 +9609,8 @@ fn build_function( ops, op_idx, guard_idx, + fail_index_base, + fail_descrs, ); let vi = op.pos().get().raw(); let can_collect = call_can_collect(op); @@ -9356,6 +9655,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9390,6 +9690,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9441,6 +9742,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9472,6 +9774,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9533,6 +9836,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9639,6 +9943,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9696,6 +10001,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9797,6 +10103,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); if !OpRef::raw_is_constant(vi) { // llmodel.py write_int_at_mem(res, vtable_offset, @@ -9852,6 +10160,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -9991,6 +10300,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -10050,6 +10360,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -10127,6 +10438,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -10192,6 +10504,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -10320,6 +10633,8 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, + gc_maps, ); // `wasm_jit_alloc_array` collects; reload other live Refs. The // inline-bump paths already emitted this inside their slow arms. @@ -10332,6 +10647,7 @@ fn build_function( residual_type_base, ca.ca_reload_fn_ptr, ca.jf_top_addr, + wb, ); emit_reload_refs_from_homes( &mut sink, @@ -11100,9 +11416,34 @@ fn unbound_pool_const_seeds( .map(|op| op.pos().get()) .collect(); let in_idx: Vec = inputargs.iter().map(|ia| ia.index).collect(); + let unresolved_raws: std::collections::HashSet = + unresolved.iter().map(|(a, _, _)| a.raw()).collect(); + let mut readers = Vec::new(); + for op in ops { + let push = |readers: &mut Vec, where_: &str, a: OpRef| { + if a == OpRef::NONE || a.is_constant() { + return; + } + if unresolved_raws.contains(&a.raw()) { + readers.push(format!("{:?} {where_} {a:?}", op.opcode)); + } + }; + for a in op.getarglist().iter() { + push(&mut readers, "arg", a.to_opref()); + } + if let Some(fa) = op.getfailargs() { + for a in fa.iter() { + push(&mut readers, "failarg", a.to_opref()); + } + } + } return Err(BackendError::Unsupported(format!( "wasm codegen: value{unresolved:?} read with no producing op and no \ - constant-pool entry; inputargs={in_idx:?} labels={labels:?} sameas={sameas:?}" + constant-pool entry; inputargs={in_idx:?} labels={labels:?} sameas={sameas:?} \ + readers=[{}] defined={} pool={}", + readers.join(", "), + defined.len(), + constants.len(), ))); } Ok(seeds) @@ -11361,6 +11702,7 @@ struct BridgeDispatch<'a> { /// later guard may spill. Keyed by value id; the pair is the baked /// table base and slot index. gc_table_slots: &'a HashMap, + gc_maps: Option<&'a FrameGcMaps>, } fn emit_guard_true( @@ -11602,6 +11944,9 @@ fn emit_guard_exit( return; } if dispatch.param_type_indices.is_empty() { + if let Some(maps) = dispatch.gc_maps { + maps.emit_exit_gcmap(sink, op, dispatch.counter_slot); + } emit_guard_spill( sink, constants, @@ -11619,6 +11964,9 @@ fn emit_guard_exit( emit_guard_param_tail_call(sink, constants, value_types, guard_idx, op, dispatch); // A missing cell keeps the historical recovery path. It is deliberately // after the cell test so a bridge crossing performs no frame spill. + if let Some(maps) = dispatch.gc_maps { + maps.emit_exit_gcmap(sink, op, dispatch.counter_slot); + } emit_guard_spill( sink, constants, @@ -11750,6 +12098,8 @@ fn emit_force_bracket_before_call( ops: &[Op], op_idx: usize, guard_idx: u32, + fail_index_base: u32, + fail_descrs: &[Arc], ) { let Some(next_op) = ops.get(op_idx + 1) else { return; @@ -11770,7 +12120,7 @@ fn emit_force_bracket_before_call( ref_homes, frame, next_op, - exit_index(next_op, guard_idx), + &fail_descrs[(guard_idx - fail_index_base) as usize], Some(ops[op_idx].pos().get().raw()), ); } @@ -11802,26 +12152,31 @@ fn emit_force_arm( ref_homes: &RefHomes, frame: FrameGeometry, guard_op: &Op, - exit_idx: u32, + descr: &Arc, undefined: Option, ) { - // `counter_value_spill` answers `None` for anything but a GUARD_VALUE, so - // the counter slot has nothing to contribute to a force bracket. - // - // Same range `emit_guard_fail_args_spill` writes and - // `normal_frame_value_slots` reserves: one past the last live position. - let mut force_args = exit_fail_args(guard_op); - force_args.truncate(live_fail_arg_extent( - guard_op.getdescr().as_ref(), - force_args.len(), - )); - for (i, &arg_ref) in force_args.iter().enumerate() { + let exit_idx = descr.fail_index; + let force_args = exit_fail_args(guard_op); + let locs = descr.rd_locs.as_ref().expect("force guard has rd_locs"); + for (i, (&arg_ref, &loc)) in force_args.iter().zip(locs).enumerate() { + if loc == 0xFFFF { + continue; + } + debug_assert_eq!( + usize::from(loc), + force_arg_location(frame, ref_homes, arg_ref, i) + ); + let is_undefined = !arg_ref.is_constant() && undefined == Some(arg_ref.raw()); sink.local_get(0); - if !arg_ref.is_constant() && undefined == Some(arg_ref.raw()) { + if is_undefined { sink.i64_const(0); } else if let Some(home) = ref_homes.home(arg_ref) { - let ofs = frame.home_slot_base + home as u64 * SLOT_SIZE; - sink.i64_const((ofs as i64) * 2 + 1); + // `dead_frame_from_forced_frame` still decodes a tagged + // home (`offset * 2 + 1`) from this force slot. The home + // itself is already stored; publish its offset so a + // collection inside the bracketed call forwards the value. + let home_offset = frame.home_slot_base + home as u64 * SLOT_SIZE; + sink.i64_const((home_offset * 2 + 1) as i64); } else { emit_resolve(sink, constants, value_types, arg_ref); } @@ -11995,10 +12350,19 @@ fn emit_memory_error_check( residual_type_base: Option, ca_reload_fn_ptr: i64, jf_top_addr: Option, + wb: &WriteBarrierHelpers, + gc_maps: &FrameGcMaps, ) { emit_resolve(sink, constants, value_types, value); sink.i64_eqz(); - emit_memory_error_on_truthy(sink, residual_type_base, ca_reload_fn_ptr, jf_top_addr); + emit_memory_error_on_truthy( + sink, + residual_type_base, + ca_reload_fn_ptr, + jf_top_addr, + wb, + gc_maps, + ); } fn emit_memory_error_if_i32_zero( @@ -12006,9 +12370,18 @@ fn emit_memory_error_if_i32_zero( residual_type_base: Option, ca_reload_fn_ptr: i64, jf_top_addr: Option, + wb: &WriteBarrierHelpers, + gc_maps: &FrameGcMaps, ) { sink.i32_eqz(); - emit_memory_error_on_truthy(sink, residual_type_base, ca_reload_fn_ptr, jf_top_addr); + emit_memory_error_on_truthy( + sink, + residual_type_base, + ca_reload_fn_ptr, + jf_top_addr, + wb, + gc_maps, + ); } fn emit_memory_error_on_truthy( @@ -12016,10 +12389,16 @@ fn emit_memory_error_on_truthy( residual_type_base: Option, ca_reload_fn_ptr: i64, jf_top_addr: Option, + wb: &WriteBarrierHelpers, + gc_maps: &FrameGcMaps, ) { sink.if_(BlockType::Empty); if crate::failguard::exit_frame_with_exception_attached() { - emit_reload_frame_if_necessary(sink, residual_type_base, ca_reload_fn_ptr, jf_top_addr); + emit_reload_frame_if_necessary(sink, residual_type_base, ca_reload_fn_ptr, jf_top_addr, wb); + gc_maps.emit_push_gcmap( + sink, + &[FRAME_SLOT_BASE as usize / std::mem::size_of::()], + ); sink.local_get(0); sink.i32_const(crate::jit_exc_value_addr() as i32); sink.i64_load(mem64(0)); @@ -12796,6 +13175,185 @@ mod tests { .expect("stale guard failarg import hole must not decline"); } + #[test] + fn gcmap_uses_live_homes_and_keeps_ca_arguments_across_frame_allocation() { + use majit_ir::forwarding::bound_operand_from_opref as rb; + let inputargs: Vec<_> = (0..3).map(|i| InputArg::from_type(Type::Ref, i)).collect(); + let first = Op::new(OpCode::CallR, &[rb(OpRef::input_arg_ref(0))]); + first.pos().set(OpRef::ref_op(3)); + let second = Op::new(OpCode::CallAssemblerR, &[rb(OpRef::input_arg_ref(1))]); + second.pos().set(OpRef::ref_op(4)); + let guard = Op::new(OpCode::GuardNotForced2, &[]); + guard.setfailargs(smallvec::smallvec![rb(OpRef::input_arg_ref(2))]); + let finish = Op::new(OpCode::Finish, &[rb(OpRef::ref_op(4))]); + let ops = vec![first, second, guard, finish]; + let live = HomeLiveness::collect_with_regions(&inputargs, &ops, &[]); + let homes = RefHomes { + by_id: vec![0, 1, 2, 3, 4], + len: 5, + }; + let maps = FrameGcMaps::new(true, FrameGeometry::compact(8, 5, 0)); + assert_eq!( + maps.live_indices(&homes, &live, 1, &ops[1]), + vec![maps.home_index(1), maps.home_index(2)] + ); + // The first call's argument and result are dead. Keeping their homes + // would retain a completed child call tree through its force token. + assert!( + !maps + .live_indices(&homes, &live, 1, &ops[1]) + .contains(&maps.home_index(3)) + ); + } + + #[test] + fn finish_gcmap_keeps_only_force_failargs_and_the_ref_result() { + use majit_ir::forwarding::bound_operand_from_opref as rb; + let homes = RefHomes { + by_id: vec![0, 1, 2], + len: 3, + }; + let maps = FrameGcMaps::new(true, FrameGeometry::compact(8, 3, 0)); + let guard = Op::new(OpCode::GuardNotForced2, &[]); + guard.setfailargs(smallvec::smallvec![rb(OpRef::input_arg_ref(1))]); + let indices = maps.force_indices(&homes, &guard); + *maps.finish_gcmap.borrow_mut() = indices; + let finish = Op::new(OpCode::Finish, &[rb(OpRef::input_arg_ref(2))]); + let mut bytes = Vec::new(); + { + let mut raw = InstructionSink::new(&mut bytes); + let mut sink = PeepSink::new(&mut raw); + maps.emit_exit_gcmap(&mut sink, &finish, None); + sink.flush(); + } + let allocated = maps.maps.borrow(); + let map = &allocated[0]; + let bits = usize::BITS as usize; + let marked: Vec<_> = (0..map[0] * bits) + .filter(|&index| map[1 + index / bits] & (1usize << (index % bits)) != 0) + .collect(); + assert_eq!( + marked, + vec![ + FRAME_SLOT_BASE as usize / std::mem::size_of::(), + maps.home_index(1) + ] + ); + // Copy, do not drain: one module can emit more than one FINISH + // (merged inline regions), and the second must keep the same + // GUARD_NOT_FORCED_2 homes. + assert_eq!(maps.finish_gcmap.borrow().as_slice(), [maps.home_index(1)]); + } + + /// x86/assembler.py `_reload_frame_if_necessary`: after a nursery frame + /// is promoted, reloading its pointer must also remember later Ref spills. + /// Keep a call result below the barrier's operands, as allocation emission + /// does, and verify the barrier neither consumes nor replaces that result. + #[test] + fn reload_frame_reapplies_write_barrier_without_clobbering_call_result() { + use wasmi::{ + Engine, Func, Linker, Memory, MemoryType, Store, Table, TableType, Val, ValType, + }; + + let wb = WriteBarrierHelpers::for_current_gc(1, 0); + let mut function = Function::new([]); + { + let mut raw = function.instructions(); + let mut sink = PeepSink::new(&mut raw); + sink.i64_const(77); + emit_reload_frame_if_necessary(&mut sink, Some(1), 0, Some(16), &wb); + sink.end(); + sink.flush(); + } + let mut types = TypeSection::new(); + types + .ty() + .function([wasm_encoder::ValType::I32], [wasm_encoder::ValType::I64]); + types.ty().function([], [wasm_encoder::ValType::I64]); + types + .ty() + .function([wasm_encoder::ValType::I64], [wasm_encoder::ValType::I64]); + let mut imports = wasm_encoder::ImportSection::new(); + imports.import( + "env", + "memory", + EntityType::Memory(wasm_encoder::MemoryType { + minimum: 1, + maximum: None, + memory64: false, + shared: false, + page_size_log2: None, + }), + ); + imports.import( + "env", + "table", + EntityType::Table(wasm_encoder::TableType { + element_type: RefType::FUNCREF, + minimum: 2, + maximum: None, + table64: false, + shared: false, + }), + ); + let mut functions = FunctionSection::new(); + functions.function(0); + let mut exports = ExportSection::new(); + exports.export("reload", ExportKind::Func, 0); + let mut code = CodeSection::new(); + code.function(&function); + let mut module = Module::new(); + module + .section(&types) + .section(&imports) + .section(&functions) + .section(&exports) + .section(&code); + let engine = Engine::default(); + let module = wasmi::Module::new(&engine, module.finish()).unwrap(); + let mut store = Store::new(&engine, Vec::::new()); + let memory = Memory::new(&mut store, MemoryType::new(1, None)).unwrap(); + let table = Table::new( + &mut store, + TableType::new(ValType::FuncRef, 2, None), + Val::default(ValType::FuncRef), + ) + .unwrap(); + let flag_addr = (128i32 + wb.flag_byteofs) as usize; + let barrier = Func::wrap( + &mut store, + move |mut caller: wasmi::Caller<'_, Vec>, frame: i64| -> i64 { + caller.data_mut().push(frame); + memory.write(&mut caller, flag_addr, &[0]).unwrap(); + 999 + }, + ); + table.set(&mut store, 1, Val::from(barrier)).unwrap(); + let mut linker = Linker::new(&engine); + linker.define("env", "memory", memory).unwrap(); + linker.define("env", "table", table).unwrap(); + let instance = linker.instantiate_and_start(&mut store, &module).unwrap(); + let reload = instance + .get_typed_func::(&store, "reload") + .unwrap(); + memory.write(&mut store, 16, &24u32.to_le_bytes()).unwrap(); + memory.write(&mut store, 20, &128u32.to_le_bytes()).unwrap(); + memory.write(&mut store, flag_addr, &[wb.if_flag]).unwrap(); + // Local 0 deliberately names a stale frame. The barrier must receive + // the forwarded object base from the shadow stack, not that old base. + assert_eq!(reload.call(&mut store, 4096).unwrap(), 77); + assert_eq!(store.data(), &[128]); + assert_eq!(reload.call(&mut store, 4096).unwrap(), 77); + assert_eq!(store.data(), &[128], "already remembered: no helper call"); + memory.write(&mut store, flag_addr, &[wb.if_flag]).unwrap(); + assert_eq!(reload.call(&mut store, 4096).unwrap(), 77); + assert_eq!( + store.data(), + &[128, 128], + "minor collection rearms the barrier" + ); + } + #[test] fn peep_sink_applies_all_local_folds() { let mut bytes = Vec::new(); @@ -12896,6 +13454,7 @@ mod tests { counter_slot: None, spill_helpers: &spill_helpers, gc_table_slots: &HashMap::new(), + gc_maps: None, }; assert_eq!(inline_region_br_depth(&inline, &dispatch, 0), 0); diff --git a/majit/majit-backend-wasm/src/failguard.rs b/majit/majit-backend-wasm/src/failguard.rs index 3959b46891d..a6de9790a4e 100644 --- a/majit/majit-backend-wasm/src/failguard.rs +++ b/majit/majit-backend-wasm/src/failguard.rs @@ -1,11 +1,11 @@ /// Guard failure descriptors and frame data for the wasm backend. /// -/// Simplified from CraneliftFailDescr — no bridge data, GC maps, or force tokens. +/// Per-emission descriptor identity and deadframe storage. use std::cell::{Cell, RefCell}; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; -use majit_ir::{Descr, DescrRef, FailDescr, Type}; +use majit_ir::{Descr, DescrRef, FailDescr, GcRef, Type}; /// Wasm-backend guard failure descriptor. #[derive(Debug)] @@ -13,6 +13,9 @@ pub struct WasmFailDescr { pub fail_index: u32, pub trace_id: u64, pub fail_arg_types: Vec, + /// llmodel.py `_decode_pos`: physical Signed-word locations for force + /// failargs. None for the normal exit adapter's positional copy. + pub rd_locs: Option>, pub is_finish: bool, /// Byte offset of GUARD_NOT_FORCED(_2)'s force-only spill area. Native /// backends carry these coordinates in their fail locations; keeping a @@ -44,6 +47,10 @@ impl Descr for WasmFailDescr { } impl FailDescr for WasmFailDescr { + fn rd_locs(&self) -> &[u16] { + self.rd_locs.as_deref().unwrap_or(&[]) + } + fn fail_index(&self) -> u32 { self.fail_index } @@ -71,6 +78,17 @@ pub struct WasmFrameData { /// exited through a GuardNoException / GuardException (0 = none), surfaced /// via `grab_exc_value`. pub exc_value: i64, + /// `JITFRAME.jf_savedata` for the wasm backend's copied deadframe. + /// The copied frame is the backend's deadframe owner, so this slot is a + /// precise host root just like the fixed GCREF field upstream. + pub savedata: i64, + /// The actual running or returned JITFRAME. + /// `llmodel.py set_savedata_ref` must publish to it, not merely to this + /// temporary copy of its exit values. The root also follows frame moves. + jitframe: GcRef, + /// Without a collector-owned JITFRAME, use the common off-GC deadframe + /// owner and its precise root walker instead of freeing the entry buffer. + host_frame: Option, /// Slots handed to [`crate::wasm_gc_add_roots`] by [`WasmFrameData::boxed`], /// released again in `Drop`. roots: Vec, @@ -79,9 +97,9 @@ pub struct WasmFrameData { impl WasmFrameData { /// `llmodel.py` reads `get_ref_value` straight out of the JITFRAME, /// which stays a GC root (its `jf_gcmap` covers the exit slots) for as long - /// as the deadframe lives. wasm has no host-visible JITFRAME to hand back: - /// `execute_token` copies the exit values into `raw_values` and drops the - /// guest frame, so the copies must carry that rooting themselves. Between + /// as the deadframe lives. wasm retains that frame and also copies the + /// exit values into `raw_values` for its exit adapter, so the copies must + /// carry that rooting themselves. Between /// the copy and the last `get_ref_value`, resume/blackhole reconstruction /// allocates freely, and a minor collection there moves exactly the objects /// these slots name. @@ -99,6 +117,9 @@ impl WasmFrameData { raw_values, fail_descr, exc_value, + savedata: 0, + jitframe: GcRef(0), + host_frame: None, roots: Vec::new(), }); let ref_count = data @@ -125,6 +146,63 @@ impl WasmFrameData { } data } + + pub fn set_savedata_ref(&mut self, value: GcRef) { + if !self.jitframe.is_null() { + if crate::wasm_gc_owns_object(self.jitframe.0) { + crate::wasm_active_gc_write_barrier(self.jitframe); + } + unsafe { + majit_backend::llmodel::set_savedata_ref( + self.jitframe.0 as *mut majit_backend::jitframe::JitFrame, + value.0, + ); + } + return; + } + let slot = &mut self.savedata as *mut i64 as usize; + if value.is_null() { + self.savedata = 0; + return; + } + if !self.roots.contains(&slot) { + unsafe { crate::wasm_gc_add_roots(&[slot]) }; + self.roots.push(slot); + } + self.savedata = value.0 as i64; + } + + pub fn get_savedata_ref(&self) -> GcRef { + if !self.jitframe.is_null() { + return GcRef(unsafe { + majit_backend::llmodel::get_savedata_ref( + self.jitframe.0 as *const majit_backend::jitframe::JitFrame, + ) + }); + } + GcRef(self.savedata as usize) + } + + /// Retain the real frame as a root, also when execute_token returns. For + /// non-GC storage the caller must hold its off-GC owner over this borrow. + pub unsafe fn attach_forced_jitframe(&mut self, frame: GcRef) { + self.jitframe = frame; + // Host-buffer frames are libc storage, not collector objects; their + // existing shadow-stack entry traces the fixed savedata field. + if crate::wasm_gc_owns_object(frame.0) { + let slot = &mut self.jitframe as *mut GcRef as usize; + unsafe { crate::wasm_gc_add_roots(&[slot]) }; + self.roots.push(slot); + } + } + + pub(crate) fn take_host_frame( + &mut self, + frame: majit_backend::libc_deadframe::LibcJitFrameDeadFrame, + ) { + self.jitframe = GcRef(frame.frame_addr()); + self.host_frame = Some(frame); + } } impl Drop for WasmFrameData { @@ -229,6 +307,7 @@ mod tests { fail_index: 0, trace_id: 0, fail_arg_types, + rd_locs: None, is_finish: false, force_args_offset: 8, force_gcmap_ptr: 0, @@ -236,6 +315,98 @@ mod tests { }) } + #[test] + fn terminal_force_reads_the_retained_host_frames_physical_locations() { + use majit_backend::jitframe::{FIRST_ITEM_OFFSET, JitFrame, alloc_off_gc_jitframe}; + use majit_backend::{Backend, DeadFrame}; + let roots = install_root_counting_gc(); + let backend = crate::WasmBackend::new(); + // x86 `store_force_descr`: `jf_force_descr` holds `index + 1`. The + // force spill sits past FINISH's result slots so `_decode_pos` still + // sees the armed values after the exit adapter overwrites items[0..2]. + let fail_index = reserve_fail_descrs(1); + let mut forced_descr = fail_descr(vec![Type::Int, Type::Float]); + { + let descr = Arc::get_mut(&mut forced_descr).unwrap(); + descr.fail_index = fail_index; + descr.force_args_offset = 16; + descr.rd_locs = Some(vec![ + (16 / std::mem::size_of::()) as u16, + (24 / std::mem::size_of::()) as u16, + ]); + } + register_fail_descrs(std::slice::from_ref(&forced_descr)); + let depth = 32 / std::mem::size_of::(); + let jf = alloc_off_gc_jitframe(JitFrame::alloc_size(depth)); + assert!(!jf.is_null()); + unsafe { JitFrame::init(jf, std::ptr::null(), depth) }; + majit_gc::shadow_stack::register_libc_jitframe(jf as usize); + let items = (jf as usize + FIRST_ITEM_OFFSET) as *mut i64; + unsafe { + (*jf).jf_force_descr = fail_index as usize + 1; + *items = 99; // FINISH's normal exit index + *items.add(1) = jf as usize as i64; // FINISH's returned token + *items.add(2) = 1i64 << 40; // must remain i64 on wasm32 + *items.add(3) = 1.25f64.to_bits() as i64; + } + let finish_descr = fail_descr(vec![Type::Ref]); + let owner = unsafe { + majit_backend::libc_deadframe::LibcJitFrameDeadFrame::owning( + jf, + jf, + depth, + finish_descr.clone(), + None, + ) + }; + let mut data = WasmFrameData::boxed(vec![jf as usize as i64], finish_descr, 0); + data.take_host_frame(owner); + let mut retained = Vec::new(); + majit_gc::walk_active_live_deadframes(&mut |addr| retained.push(addr)); + assert!(retained.contains(&(jf as usize))); + let returned = DeadFrame::Boxed(data); + let token = backend.get_ref_value(&returned, 0); + assert!(backend.is_force_token_armed(token)); + let forced = backend.force(token).unwrap(); + assert_eq!(backend.get_int_value(&forced, 0), 1i64 << 40); + assert_eq!(backend.get_float_value(&forced, 1), 1.25); + assert_ne!( + unsafe { *items } & crate::codegen::FORCE_TAKEN_BIT, + 0, + "force marks frame[0] so GUARD_NOT_FORCED deopts" + ); + drop(forced); + assert!(majit_gc::shadow_stack::is_libc_jitframe(jf as usize)); + drop(returned); + assert!(!majit_gc::shadow_stack::is_libc_jitframe(jf as usize)); + assert_eq!(roots.load(Ordering::SeqCst), 0); + } + + #[test] + fn forcing_publishes_savedata_on_the_running_frame() { + let roots = install_root_counting_gc(); + use majit_backend::jitframe::{JitFrame, alloc_off_gc_jitframe, free_off_gc_jitframe}; + let jf = alloc_off_gc_jitframe(JitFrame::alloc_size(0)); + assert!(!jf.is_null()); + unsafe { JitFrame::init(jf, std::ptr::null(), 0) }; + { + let mut forced = WasmFrameData::boxed(vec![], fail_descr(vec![]), 0); + unsafe { forced.attach_forced_jitframe(GcRef(jf as usize)) }; + forced.set_savedata_ref(GcRef(0x1000)); + assert_eq!(forced.get_savedata_ref(), GcRef(0x1000)); + } + // The force-time DeadFrame wrapper is gone; the subsequently failing + // guard must still find the cache on the actual JITFRAME. + assert_eq!(unsafe { (*jf).jf_savedata }, 0x1000); + let mut exited = WasmFrameData::boxed(vec![], fail_descr(vec![]), 0); + exited.set_savedata_ref(GcRef(unsafe { (*jf).jf_savedata })); + unsafe { free_off_gc_jitframe(jf) }; + assert_eq!(exited.get_savedata_ref(), GcRef(0x1000)); + assert_eq!(roots.load(Ordering::SeqCst), 1); + drop(exited); + assert_eq!(roots.load(Ordering::SeqCst), 0); + } + #[test] fn a_finish_singleton_resolves_to_its_reserved_exit() { // The emitted FINISH writes the index this returns and the emitted @@ -286,6 +457,7 @@ mod tests { fail_index: base + i, trace_id: 0, fail_arg_types: vec![Type::Ref], + rd_locs: None, is_finish: false, force_args_offset: 8, force_gcmap_ptr: 0, @@ -320,6 +492,7 @@ mod tests { fail_index: base + index as u32, trace_id: trace_id as u64, fail_arg_types: vec![Type::Int], + rd_locs: None, is_finish: false, force_args_offset: 8, force_gcmap_ptr: 0, @@ -583,6 +756,7 @@ fn reserved_finish_descr(exit_index: u32, meta_descr: Option) -> Arc>>, + /// Assembler descriptor pointers embedded in current and retired code. + pub force_descrs: RefCell>>, /// Owning `JitCellToken` number, used to retract this loop's /// CALL_ASSEMBLER target metadata on drop. pub token_number: u64, diff --git a/majit/majit-backend-wasm/src/lib.rs b/majit/majit-backend-wasm/src/lib.rs index 1eb4df92342..f680833d5a5 100644 --- a/majit/majit-backend-wasm/src/lib.rs +++ b/majit/majit-backend-wasm/src/lib.rs @@ -1168,6 +1168,12 @@ fn with_wasm_active_gc_mut(f: impl FnOnce(&mut dyn GcAllocator) -> R) -> Opti /// box in TLS) and `install_gc_standalone` (production: hooks only, no box /// — the trampolines then route to the `gc_sync` singleton). fn register_active_hooks(supports_guard_gc_type: bool) { + // llmodel.py execute_token returns the live JITFRAME as its deadframe. + // The host-entry fallback is off-GC, so its retained frame needs the + // common precise walker after it leaves the JF shadow stack. + majit_gc::set_active_gc_deadframe_hooks(majit_gc::ActiveGcDeadFrameHooks { + walk_live_deadframes: Some(majit_backend::libc_deadframe::walk_live_deadframes), + }); majit_gc::set_active_gc_guard_hooks(majit_gc::ActiveGcGuardHooks { check_is_object: Some(wasm_check_is_object), is_tagged_immediate: Some(wasm_is_tagged_immediate), @@ -1193,6 +1199,9 @@ fn register_active_hooks(supports_guard_gc_type: bool) { majit_gc::set_active_alloc_nursery_collecting_typed_rooted(Some( wasm_alloc_nursery_collecting_typed_rooted, )); + majit_gc::set_active_alloc_nursery_collecting_typed_roots(Some( + wasm_alloc_nursery_collecting_typed_roots, + )); majit_gc::set_active_alloc_oldgen_typed(Some(wasm_alloc_oldgen_typed)); majit_gc::set_active_root_hooks(Some(wasm_gc_add_root), Some(wasm_gc_remove_root)); majit_gc::set_active_gc_owns_object(Some(wasm_gc_owns_object)); @@ -1382,14 +1391,12 @@ fn ca_inline_params(frame_bytes: u32) -> Option { /// Whether the host entry runs a trace on a `JitFrame` it pushed onto the /// jitframe shadow stack. /// -/// `execute_token` allocates that frame only once a `JitFrame` type id has been -/// registered; with none it runs the trace on a plain host buffer, which no -/// collection moves and which the shadow stack never describes. Every frame -/// reload a trace body emits answers out of that shadow stack, so an embedder -/// that registered no type id must get no reloads at all — a reload there would -/// replace the running frame pointer with whatever root happens to sit on top. +/// Both wasm execute_token paths push a real header: the collector-owned +/// frame, or the common off-GC JITFRAME allocation. The latter never moves but +/// still needs the force/exit gcmap and remains described by the same stack. +/// Native codegen-only tests have no host entry unless they install a GC. fn host_entry_frame_is_jitframe() -> bool { - wasm_jitframe_tid() != 0 + cfg!(target_arch = "wasm32") || wasm_jitframe_tid() != 0 } /// Address of the active jitframe shadow-stack top cell for ordinary trace @@ -1654,6 +1661,25 @@ unsafe fn wasm_alloc_nursery_collecting_typed_rooted( .unwrap_or(GcRef(0)) } +unsafe fn wasm_alloc_nursery_collecting_typed_roots( + type_id: u32, + size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, +) -> GcRef { + with_wasm_active_gc_mut(|gc| unsafe { + gc.alloc_fast_nursery_collecting_typed_roots( + type_id, + size, + roots, + root_count, + needs_write_barrier, + ) + }) + .unwrap_or(GcRef(0)) +} + /// Host-side old-gen allocation trampoline. Stable /// across minor/major collections — see dynasm counterpart. fn wasm_alloc_oldgen_typed(type_id: u32, size: usize) -> GcRef { @@ -3295,13 +3321,15 @@ impl WasmBackend { // Key-0 still clears the full used-label range. inputs.ca.home_gcmap_min_ordinary = compiled.num_ref_homes.get(); inputs.ca.home_gcmap_min_labels = compiled.used_label_homes.get(); - let (wasm_bytes, guard_exits, merged_ref_homes, merged_labels) = - codegen::build_wasm_module(&inputs)?; + let (wasm_bytes, guard_exits, module_data) = codegen::build_wasm_module(&inputs)?; + let merged_ref_homes = module_data.num_ref_homes; + let merged_labels = module_data.used_label_homes; let code_size = wasm_bytes.len(); let descrs: Vec> = guard_exits .iter() + .zip(&module_data.fail_descrs) .enumerate() - .map(|(index, g)| { + .map(|(index, (g, descr))| { let mut region_start = own_guard_count; let trace_id = inputs .inlined_bridges @@ -3317,6 +3345,7 @@ impl WasmBackend { fail_index: g.fail_index, trace_id, fail_arg_types: g.fail_arg_types.clone(), + rd_locs: descr.rd_locs.clone(), is_finish: g.is_finish, force_args_offset: inputs.frame.force_slot_base as u32, force_gcmap_ptr: leak_gcmap_for_indices(&g.force_ref_home_indices), @@ -3375,6 +3404,11 @@ impl WasmBackend { // in the same lifetime ledger as an ordinary compiled module. let block = self.asm_memory_stats.record_block(code_size, code_size); self.asm_memory_blocks.push(block); + compiled.gc_maps.borrow_mut().extend(module_data.gc_maps); + compiled + .force_descrs + .borrow_mut() + .extend(descrs.iter().filter(|d| d.rd_locs.is_some()).cloned()); // Keep still-standalone bridge descriptors after the rebuilt merged // prefix. Adding regions grows that prefix, so every old positional // range moves by exactly the difference in guard-cell counts. @@ -3984,13 +4018,16 @@ pub fn dead_frame_from_ran_frame(_compiled_ptr: usize, frame_ptr: usize) -> Dead let raw_values: Vec = (0..num_outputs) .map(|i| unsafe { *frame.add(1 + i) }) .collect(); - DeadFrame::Boxed(WasmFrameData::boxed(raw_values, fail_descr, exc_value)) + let mut data = WasmFrameData::boxed(raw_values, fail_descr, exc_value); + // `boxed` may collect and forward a nursery callee. Reload the object + // base from the JF shadow stack before reading `jf_savedata`. + let jf = jitframe_object_base(frame_ptr).0 as *const majit_backend::jitframe::JitFrame; + data.set_savedata_ref(GcRef(unsafe { (*jf).jf_savedata })); + DeadFrame::Boxed(data) } -/// Reconstruct a [`DeadFrame`] for a frame a FORCE interrupted while its call -/// is still on the stack, from the coordinate `emit_force_bracket_before_call` -/// published into it: `frame[0]` the bracketing GUARD_NOT_FORCED's exit index, -/// `frame[1..]` that guard's fail arguments. +/// llmodel.py force / `_decode_pos`: read the forced descriptor's saved +/// physical locations, including after FINISH has overwritten its result slot. /// /// Twin of [`dead_frame_from_ran_frame`] with one difference: a force is not an /// exit, so it must not consume the pending-exception cell. `jit_exc_take` @@ -4003,6 +4040,18 @@ fn forced_frame_items_base(force_token: GcRef) -> usize { force_token.0 + majit_backend::jitframe::FIRST_ITEM_OFFSET } +/// JitFrame object base for a data-region `frame_ptr`. After a collecting +/// `WasmFrameData::boxed`, prefer the forwarded address on the JF shadow +/// stack; tests that never pushed a frame keep the incoming pointer. +fn jitframe_object_base(frame_ptr: usize) -> GcRef { + let top = majit_gc::shadow_stack::jf_top_ptr(); + if top.is_null() { + GcRef(frame_ptr - majit_backend::jitframe::FIRST_ITEM_OFFSET) + } else { + top + } +} + fn force_arg_word(frame_ptr: usize, fail_descr: &WasmFailDescr, index: usize) -> i64 { let offset = fail_descr.force_args_offset as usize + index * std::mem::size_of::(); unsafe { *((frame_ptr + offset) as *const i64) } @@ -4030,7 +4079,15 @@ fn dead_frame_from_forced_frame(frame_ptr: usize, fail_index: u32) -> DeadFrame value }) .collect(); - DeadFrame::Boxed(WasmFrameData::boxed(raw_values, fail_descr, 0)) + let mut data = WasmFrameData::boxed(raw_values, fail_descr, 0); + unsafe { + // `boxed` may collect. Attach the forwarded object base, not the + // incoming items pointer. `attach_forced_jitframe` wants that + // object base: it checks GC ownership and later casts it to + // `*mut JitFrame` in `set_savedata_ref`. + data.attach_forced_jitframe(jitframe_object_base(frame_ptr)); + } + DeadFrame::Boxed(data) } /// Install the recovery guard's compile-time map before dropping the execution @@ -4427,7 +4484,7 @@ impl majit_backend::Backend for WasmBackend { }, ), }; - let (wasm_bytes, guard_exits, num_ref_homes, _used_labels) = + let (wasm_bytes, guard_exits, module_data) = match codegen::build_wasm_module(&module_inputs) { Ok(built) => built, Err(err) => { @@ -4436,16 +4493,19 @@ impl majit_backend::Backend for WasmBackend { return Err(err); } }; + let num_ref_homes = module_data.num_ref_homes; let home_gcmap_ptr = leak_home_gcmap(frame, num_ref_homes, used_label_homes); // Build fail descriptors let fail_descrs: Vec> = guard_exits .iter() - .map(|g| { + .zip(&module_data.fail_descrs) + .map(|(g, descr)| { Arc::new(WasmFailDescr { fail_index: g.fail_index, trace_id, fail_arg_types: g.fail_arg_types.clone(), + rd_locs: descr.rd_locs.clone(), is_finish: g.is_finish, force_args_offset: frame.force_slot_base as u32, force_gcmap_ptr: leak_gcmap_for_indices(&g.force_ref_home_indices), @@ -4551,6 +4611,14 @@ impl majit_backend::Backend for WasmBackend { // scoped to one `execute_token` call. let compiled = CompiledWasmLoop { + gc_maps: std::cell::RefCell::new(module_data.gc_maps), + force_descrs: std::cell::RefCell::new( + fail_descrs + .iter() + .filter(|d| d.rd_locs.is_some()) + .cloned() + .collect(), + ), token_number: token.number, trace_id, input_types: inputargs.iter().map(|ia| ia.tp).collect(), @@ -4561,8 +4629,10 @@ impl majit_backend::Backend for WasmBackend { fail_descrs: std::cell::RefCell::new(fail_descrs), num_inputs: inputargs.len(), max_output_slots, - num_ref_homes: std::cell::Cell::new(num_ref_homes), - used_label_homes: std::cell::Cell::new(used_label_homes), + num_ref_homes: std::cell::Cell::new(module_data.num_ref_homes), + used_label_homes: std::cell::Cell::new( + used_label_homes.max(module_data.used_label_homes), + ), frame, home_gcmap_ptr: std::cell::Cell::new(home_gcmap_ptr), bridge_cells_base: std::cell::Cell::new(bridge_cells_base), @@ -5462,7 +5532,7 @@ impl majit_backend::Backend for WasmBackend { frame: source_frame, ca: ca_params, }; - let (wasm_bytes, guard_exits, _num_ref_homes, _used_labels) = + let (wasm_bytes, guard_exits, module_data) = match codegen::build_wasm_module(&module_inputs) { Ok(built) => built, Err(err) => { @@ -5474,11 +5544,13 @@ impl majit_backend::Backend for WasmBackend { // Bridge exit descrs (fail_index already base-offset by build_wasm_module). let bridge_descrs: Vec> = guard_exits .iter() - .map(|g| { + .zip(&module_data.fail_descrs) + .map(|(g, descr)| { Arc::new(WasmFailDescr { fail_index: g.fail_index, trace_id, fail_arg_types: g.fail_arg_types.clone(), + rd_locs: descr.rd_locs.clone(), is_finish: g.is_finish, force_args_offset: source_frame.force_slot_base as u32, force_gcmap_ptr: leak_gcmap_for_indices(&g.force_ref_home_indices), @@ -5551,6 +5623,13 @@ impl majit_backend::Backend for WasmBackend { .get() .and_then(|c| c.downcast_ref::()) .expect("source loop disappeared between borrows"); + source_loop.gc_maps.borrow_mut().extend(module_data.gc_maps); + source_loop.force_descrs.borrow_mut().extend( + bridge_descrs + .iter() + .filter(|d| d.rd_locs.is_some()) + .cloned(), + ); // Append the bridge's exit descrs to the source loop's flat // `fail_descrs` and record the slice they occupy, keyed by the // source guard's `fail_index`. `compiled_bridge_fail_descr_layouts` @@ -5912,6 +5991,9 @@ impl majit_backend::Backend for WasmBackend { } let saved = majit_gc::shadow_stack::push_jf(jf_ref); + // assembler.py `_reload_frame_if_necessary`: the host entry + // frame is old-generation before its first Ref-home spill. + wasm_active_gc_write_barrier(jf_ref); glue::execute(func_handle, items_base as u32); let exc_value = jit_exc_take(); @@ -5931,9 +6013,20 @@ impl majit_backend::Backend for WasmBackend { // virtualizable token is an independent edge to this JITFRAME; // its lazy force may arrive after the execution root is gone. install_post_finish_force_gcmap(jf); - remember_and_drop_execution_frame(jf, saved); - - return DeadFrame::Boxed(WasmFrameData::boxed(raw_values, fail_descr, exc_value)); + // Main's `remember_and_drop_execution_frame` writes the + // barrier then drops. Keep the JITFRAME rooted across + // `boxed()` so `jf_savedata` forwards, and apply the + // barrier now; the drop is after `set_savedata_ref`. + wasm_jit_write_barrier(jf as i64); + let mut data = WasmFrameData::boxed(raw_values, fail_descr, exc_value); + // `boxed` registers the copied Ref slots and may collect. + // Keep the JITFRAME on the shadow stack across that call so + // `jf_savedata` is forwarded, then publish the updated + // address before dropping the frame root. + let jf = majit_gc::shadow_stack::peek_jf(saved).0 as *mut JitFrame; + data.set_savedata_ref(GcRef(unsafe { (*jf).jf_savedata })); + majit_gc::shadow_stack::pop_jf_to(saved); + return DeadFrame::Boxed(data); } // Host-buffer frame path, for an embedder that registered no @@ -5941,8 +6034,7 @@ impl majit_backend::Backend for WasmBackend { // item[1 + i], surviving Ref homes rooted across the trace. A home // slot only ever holds null (entry init) or a valid GcRef // (store-on-def), so forwarding is safe. No collection moves this - // buffer, which is what `host_entry_frame_is_jitframe` reports to - // codegen so the body emits no frame reload. The release below is + // buffer; a body reload simply reads the same stack root. The release below is // straight-line and the wasm32 build is `panic=abort`, so // `glue::execute` cannot unwind and leak roots. // @@ -5961,10 +6053,11 @@ impl majit_backend::Backend for WasmBackend { let sign = std::mem::size_of::(); let depth = frame_size * 8 / sign; let alloc_size = majit_backend::jitframe::JitFrame::alloc_size(depth); - // An `i64` element type for the alignment a `JitFrame` needs and - // for the zero fill `JitFrame::init` requires. - let mut backing = vec![0i64; alloc_size.div_ceil(8)]; - let jf = backing.as_mut_ptr() as *mut majit_backend::jitframe::JitFrame; + // Off-GC storage so a FINISH that returns the force token can + // hand the same block to `LibcJitFrameDeadFrame::owning`. A + // `Vec` on this stack would free the token's JitFrame. + let jf = majit_backend::jitframe::alloc_off_gc_jitframe(alloc_size); + assert!(!jf.is_null(), "wasm host-buffer JitFrame allocation failed"); unsafe { majit_backend::jitframe::JitFrame::init(jf, std::ptr::null(), depth) }; unsafe { (*jf).jf_gcmap = compiled.home_gcmap_ptr.get() as *const u8 }; let items = (jf as usize + majit_backend::jitframe::FIRST_ITEM_OFFSET) as *mut i64; @@ -5989,9 +6082,10 @@ impl majit_backend::Backend for WasmBackend { glue::execute(func_handle, items as usize as u32); } majit_gc::shadow_stack::pop_jf_to(saved); - majit_gc::shadow_stack::unregister_libc_jitframe(jf as usize); // Nothing reads the frame's interior through the gcmap any more, - // and the gcmap is about to go out of scope. + // and the gcmap is about to go out of scope. Keep the libc + // registration: `owning` walks this frame as a deadframe root + // and unregisters it on drop. unsafe { (*jf).jf_gcmap = std::ptr::null() }; for h in 0..compiled.frame.home_slots { let slot = unsafe { items.add(home_base + h) } as *mut GcRef; @@ -6006,8 +6100,21 @@ impl majit_backend::Backend for WasmBackend { let raw_values: Vec = (0..num_outputs) .map(|i| unsafe { *items.add(1 + i) }) .collect(); - drop(backing); - DeadFrame::Boxed(WasmFrameData::boxed(raw_values, fail_descr, exc_value)) + // FINISH(force_token) parks this JitFrame pointer in raw_values. + // Own the off-GC block before `boxed` so a later `force` does + // not dereference a freed frame. + let owner = unsafe { + majit_backend::libc_deadframe::LibcJitFrameDeadFrame::owning( + jf, + jf, + depth, + fail_descr.clone(), + None, + ) + }; + let mut data = WasmFrameData::boxed(raw_values, fail_descr, exc_value); + data.take_host_frame(owner); + DeadFrame::Boxed(data) } } @@ -6090,6 +6197,23 @@ impl majit_backend::Backend for WasmBackend { GcRef(data.exc_value as usize) } + fn set_savedata_ref(&self, frame: &mut DeadFrame, value: GcRef) { + let data = frame + .boxed_data_mut() + .and_then(|d| d.downcast_mut::()) + .expect("not WasmFrameData"); + data.set_savedata_ref(value); + } + + fn get_savedata_ref(&self, frame: &DeadFrame) -> Option { + let data = frame + .boxed_data() + .and_then(|d| d.downcast_ref::()) + .expect("not WasmFrameData"); + let value = data.get_savedata_ref(); + (!value.is_null()).then_some(value) + } + fn clear_stored_exception(&self) { crate::jit_exc_clear(); } diff --git a/majit/majit-backend-wasm/tests/codegen_test.rs b/majit/majit-backend-wasm/tests/codegen_test.rs index c8828d3b285..1efe4d8322a 100644 --- a/majit/majit-backend-wasm/tests/codegen_test.rs +++ b/majit/majit-backend-wasm/tests/codegen_test.rs @@ -386,9 +386,9 @@ fn recursive_call_assembler_does_not_refill_zeroed_nursery_frames() { // same event `bridges_compiled` counts, since both are bumped only on the // `Ok` side of `compile_bridge`. So both follow from the committed // `pyre/bench/fib_recursive.wasm.jitstats`: `loops_compiled=1` + - // `bridges_compiled=7`. Re-record these two alongside that baseline. - assert_eq!(stat_value(&stderr, "compiles"), 8); - assert_eq!(stat_value(&stderr, "BRIDGE_OK"), 7); + // `bridges_compiled=3`. Re-record these two alongside that baseline. + assert_eq!(stat_value(&stderr, "compiles"), 4); + assert_eq!(stat_value(&stderr, "BRIDGE_OK"), 3); assert_no_call_assembler_frame_fill(&stderr); } @@ -594,14 +594,15 @@ fn build_module( vtable_offset: Option, gc_info: &codegen::GuardGcTypeInfo, ) -> (Vec, Vec) { - build_module_with_frame( - inputargs, - ops, - constants, - vtable_offset, - gc_info, - codegen::FrameGeometry::fixed(), - ) + let homes = codegen::count_ref_homes(inputargs, ops); + let frame = if homes == 0 { + codegen::FrameGeometry::fixed() + } else { + let values = codegen::frame_value_slots(inputargs, ops) + .max(codegen::FrameGeometry::fixed().value_slots); + codegen::FrameGeometry::compact(values, homes, 0) + }; + build_module_with_frame(inputargs, ops, constants, vtable_offset, gc_info, frame) } fn build_module_with_frame( @@ -658,7 +659,7 @@ fn build_module_with_ca( frame, ca, }; - let (bytes, guards, _, _) = + let (bytes, guards, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); (bytes, guards) } @@ -1081,8 +1082,7 @@ fn build_module_with_write_barrier_target( frame: codegen::FrameGeometry::compact(5, 2, 0), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); bytes } @@ -1710,7 +1710,7 @@ fn test_cold_guard_recovery_preserves_nonzero_base_and_typed_bits() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, guards, _, _) = + let (bytes, guards, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); assert_eq!(guards[0].fail_index, FAIL_INDEX_BASE); @@ -1785,7 +1785,7 @@ fn test_empty_trace() { /// slots. Every inline-region repro drives the module exactly this way and /// differs only in the exit index it expects. fn run_inline_region_trace(inputs: &codegen::ModuleBuildInputs) -> (i64, i64, i64, i64) { - let (bytes, _, _, _) = codegen::build_wasm_module(inputs).expect("non-header region merges"); + let (bytes, _, _) = codegen::build_wasm_module(inputs).expect("non-header region merges"); validate_wasm(&bytes); let engine = Engine::default(); @@ -1863,7 +1863,7 @@ fn a_deferred_merge_trips_once_at_its_threshold() { dispatch_cell_index: CELL_INDEX, }); - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs).expect("the armed module builds"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("the armed module builds"); validate_wasm(&bytes); let engine = Engine::default(); @@ -1976,6 +1976,71 @@ fn inline_region_inputs( } } +#[test] +fn terminal_force_descriptor_and_failargs_survive_finish() { + // runner_test.py test_guard_not_forced_2: return FORCE_TOKEN, then force + // the completed frame and read the guard's original failargs. + let inputargs = vec![ + InputArg::from_type(Type::Int, 0), + InputArg::from_type(Type::Int, 1), + ]; + let sum = make_op( + OpCode::IntAdd, + &[OpRef::input_arg_int(0), OpRef::input_arg_int(1)], + OpRef::int_op(2), + ); + let token = make_op(OpCode::ForceToken, &[], OpRef::ref_op(3)); + let guard = make_guard(OpCode::GuardNotForced2, &[], &[OpRef::int_op(2)]); + let finish = Op::new(OpCode::Finish, &[rb(OpRef::ref_op(3))]); + let inputs = inline_region_inputs(&inputargs, vec![sum, token, guard, finish], vec![]); + let (bytes, exits, data) = codegen::build_wasm_module(&inputs).unwrap(); + let engine = Engine::default(); + let module = Module::new(&engine, bytes).unwrap(); + let mut store = Store::new(&engine, ()); + let memory = Memory::new(&mut store, MemoryType::new(1, None)).unwrap(); + let frame = 128usize; + let items = frame + majit_backend::jitframe::FIRST_ITEM_OFFSET; + memory + .write(&mut store, items + 8, &20i64.to_le_bytes()) + .unwrap(); + memory + .write(&mut store, items + 16, &10i64.to_le_bytes()) + .unwrap(); + let mut linker = Linker::new(&engine); + linker.define("env", "memory", memory).unwrap(); + let instance = linker.instantiate_and_start(&mut store, &module).unwrap(); + instance + .get_typed_func::(&store, "trace") + .unwrap() + .call(&mut store, items as i32) + .unwrap(); + let read_i64 = |offset| { + let mut bytes = [0; 8]; + memory.read(&store, offset, &mut bytes).unwrap(); + i64::from_le_bytes(bytes) + }; + assert_eq!(read_i64(items) as u32, exits[1].fail_index); + assert_eq!(read_i64(items + 8), frame as i64); + let force = &data.fail_descrs[0]; + let mut ptr_bytes = [0; 4]; + memory + .read( + &store, + frame + majit_backend::jitframe::JF_FORCE_DESCR_OFS as usize, + &mut ptr_bytes, + ) + .unwrap(); + // x86 `store_force_descr` keeps the descriptor in `jf_force_descr`. + // wasm stores `index + 1`; zero remains the unarmed sentinel. + assert_eq!( + u32::from_le_bytes(ptr_bytes), + force.fail_index.wrapping_add(1) + ); + // x86/regalloc.py `consider_guard_not_forced_2`: force failargs sit + // in the dedicated spill, not in FINISH's return slot. + assert_eq!(read_i64(items + force.force_args_offset as usize), 30); +} + #[test] fn inlined_bridge_without_owner_loop_label_declines() { let inputargs = vec![InputArg::from_type(Type::Int, 0)]; @@ -2146,7 +2211,7 @@ fn inlined_bridge_carrying_an_unarmed_call_assembler_declines() { frame: codegen::FrameGeometry::fixed(), ca, }; - codegen::build_wasm_module(&inputs).map(|(bytes, _, _, _)| bytes) + codegen::build_wasm_module(&inputs).map(|(bytes, _, _)| bytes) } /// The region the decline arms use, with `opcode` producing its one value. @@ -2724,7 +2789,7 @@ fn inlined_bridge_emission_is_independent_of_the_regions_own_numbering() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - codegen::build_wasm_module(&inputs).map(|(bytes, _, _, _)| bytes) + codegen::build_wasm_module(&inputs).map(|(bytes, _, _)| bytes) } let colliding = build(2); @@ -2986,7 +3051,7 @@ fn compute_home_gcmap_simple_loop_is_valid() { frame, ca, }; - let (bytes, _, _, _) = + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("compute_home_gcmap loop should build"); validate_wasm(&bytes); } @@ -3056,7 +3121,7 @@ fn home_gcmap_union_call_validates_on_a_reload_only_residual_family() { frame, ca, }; - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs) + let (bytes, _, _) = codegen::build_wasm_module(&inputs) .expect("reload-only residual family must declare arity 2 for union"); validate_wasm(&bytes); } @@ -3130,7 +3195,7 @@ fn home_gcmap_publish_pointer_eq_guards_the_union_call() { frame, ca, }; - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs) + let (bytes, _, _) = codegen::build_wasm_module(&inputs) .expect("cover-check publish must still declare the union residual"); validate_wasm(&bytes); let mut ptr_eqs = 0usize; @@ -3204,10 +3269,10 @@ fn reemit_nulls_grown_label_homes_and_builds() { frame, ca, }; - let (bytes, _, _, used_labels) = + let (bytes, _, data) = codegen::build_wasm_module(&inputs).expect("re-emission grown LABEL tail should build"); validate_wasm(&bytes); - assert!(used_labels >= 1); + assert!(data.used_label_homes >= 1); } /// A peeled loop's start LABEL can still name an import_state source @@ -3266,7 +3331,7 @@ fn label_arg_import_hole_is_not_an_unbound_read() { frame, ca, }; - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs) + let (bytes, _, _) = codegen::build_wasm_module(&inputs) .expect("stale LABEL import hole must not decline the module"); validate_wasm(&bytes); } @@ -3547,7 +3612,7 @@ fn zero_arity_parameter_entry_is_structurally_type_zero() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs).unwrap(); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).unwrap(); validate_wasm(&bytes); assert_eq!( @@ -4080,7 +4145,7 @@ fn test_guard_not_invalidated_loads_runtime_flag() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, guards, _, _) = + let (bytes, guards, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); @@ -4463,8 +4528,7 @@ fn gc_table_load_inside_a_loop_body_is_emitted_inside_the_loop() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); let mut control_stack = Vec::new(); @@ -4576,8 +4640,7 @@ fn preamble_gc_table_failarg_is_reloaded_inside_the_loop() { frame: codegen::FrameGeometry::compact(5, 3, 1), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); let mut control_stack = Vec::new(); @@ -4911,7 +4974,7 @@ fn build_external_jump_module( frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, guards, _, _) = + let (bytes, guards, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); (bytes, guards) } @@ -5129,7 +5192,7 @@ fn build_owner_with_region_closing_at( constants: indexmap::IndexMap::new(), }], ); - codegen::build_wasm_module(&inputs).map(|(bytes, _, _, _)| bytes) + codegen::build_wasm_module(&inputs).map(|(bytes, _, _)| bytes) } /// A region closing at the loop HEADER `br`s to the `loop`, the long-standing @@ -5405,7 +5468,7 @@ fn test_non_moving_descr_allocates_through_the_oldgen_helper() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); const_immediates(&bytes) @@ -6455,7 +6518,7 @@ fn region_closing_at_the_header_permutes_two_ref_label_args() { frame: codegen::FrameGeometry::fixed(), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs).expect("permuting region merges"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("permuting region merges"); validate_wasm(&bytes); let engine = Engine::default(); @@ -6661,7 +6724,7 @@ fn run_header_region_repro(full_arity: bool, region_guard: RegionGuard) { ); return; } - let (bytes, _, _, _) = built.expect("header region merges"); + let (bytes, _, _) = built.expect("header region merges"); validate_wasm(&bytes); let engine = Engine::default(); @@ -6983,8 +7046,7 @@ fn build_module_with_barrier_helpers( frame: codegen::FrameGeometry::compact(6, 3, 0), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("write barrier module compiles"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("write barrier module compiles"); validate_wasm(&bytes); bytes } @@ -8239,7 +8301,7 @@ fn threadlocalref_get_lowers_through_the_tls_helper() { ); inputs.inputargs = vec![InputArg::from_type(Type::Int, 0)]; inputs.alloc.threadlocal_fn_ptr = 0x66; - let (bytes, _, _, _) = + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("ThreadlocalrefGet should lower"); validate_wasm(&bytes); @@ -8372,9 +8434,9 @@ fn consecutive_allocations_home_and_reload_before_each_collection() { large_threshold: 4096, plain_tids: [53].into_iter().collect(), }); - let (bytes, _, homes, _) = codegen::build_wasm_module(&inputs).unwrap(); + let (bytes, _, data) = codegen::build_wasm_module(&inputs).unwrap(); assert_eq!( - homes, 2, + data.num_ref_homes, 2, "the first two objects cross a subsequent allocation" ); validate_wasm(&bytes); @@ -8537,8 +8599,7 @@ fn consecutive_new_ops_share_one_nursery_bump() { vec![plain_new(1, 53), plain_new(2, 53), finish_int_arg0()], 53, ); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8557,8 +8618,7 @@ fn news_that_fill_the_nursery_threshold_keep_separate_bumps() { ], 53, ); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8592,8 +8652,7 @@ fn inlined_region_new_does_not_join_the_owners_nursery_batch() { gc_table_base: 0, constants: indexmap::IndexMap::new(), }]; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8620,8 +8679,7 @@ fn collecting_op_between_news_keeps_separate_nursery_bumps() { vec![plain_new(1, 53), call, plain_new(2, 53), finish_int_arg0()], 53, ); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8652,8 +8710,7 @@ fn setfield_between_news_keeps_one_nursery_bump() { let finish = Op::new(OpCode::Finish, &[rb(OpRef::input_arg_int(1))]); finish.setfailargs(smallvec![rb(OpRef::input_arg_int(1))]); inputs.ops[3] = finish; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8675,8 +8732,7 @@ fn new_and_const_newarray_share_one_nursery_bump() { if let Some(na) = inputs.nursery.as_mut() { na.plain_tids.insert(55); } - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8695,8 +8751,7 @@ fn consecutive_const_newarrays_share_one_nursery_bump() { ], 55, ); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8729,8 +8784,7 @@ fn runtime_newarray_flushes_the_nursery_batch() { if let Some(na) = inputs.nursery.as_mut() { na.plain_tids.insert(55); } - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8750,8 +8804,7 @@ fn call_malloc_nursery(result: u32, size: i64) -> Op { #[test] fn call_malloc_nursery_uses_one_inline_bump() { let inputs = nursery_new_inputs(vec![call_malloc_nursery(1, 32), finish_int_arg0()], 53); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!(nursery_top_compare_count(&bytes), 1); } @@ -8759,8 +8812,7 @@ fn call_malloc_nursery_uses_one_inline_bump() { #[test] fn inline_nursery_new_zeros_its_payload() { let inputs = nursery_new_inputs(vec![plain_new(1, 53), finish_int_arg0()], 53); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); let mut fills = 0; count_operators(&bytes, |op| { @@ -8785,8 +8837,7 @@ fn call_malloc_nursery_and_ptr_increment_share_one_bump() { vec![call_malloc_nursery(1, 72), incr, finish_int_arg0()], 53, ); - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( nursery_top_compare_count(&bytes), @@ -8813,7 +8864,7 @@ fn call_malloc_nursery_variants_lower() { OpRef::ref_op(1), ); let inputs = nursery_new_inputs(vec![headerless, finish_int_arg0()], 53); - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs).expect("headerless should lower"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("headerless should lower"); validate_wasm(&bytes); assert_eq!(nursery_top_compare_count(&bytes), 1); assert!(const_immediates(&bytes).contains(&0x55)); @@ -8824,7 +8875,7 @@ fn call_malloc_nursery_variants_lower() { OpRef::ref_op(1), ); let inputs = nursery_new_inputs(vec![headerless_odd, finish_int_arg0()], 53); - let (bytes, _, _, _) = + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("unaligned headerless should lower"); validate_wasm(&bytes); assert!( @@ -8838,7 +8889,7 @@ fn call_malloc_nursery_variants_lower() { OpRef::ref_op(1), ); let inputs = nursery_new_inputs(vec![frame, finish_int_arg0()], 53); - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs).expect("varsize frame should lower"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("varsize frame should lower"); validate_wasm(&bytes); assert_eq!(nursery_top_compare_count(&bytes), 1); assert!(const_immediates(&bytes).contains(&0x11)); @@ -8858,7 +8909,7 @@ fn call_malloc_nursery_variants_lower() { ); varsize.setdescr(Arc::new(SimpleArrayDescr::new(1, 16, 8, 53, Type::Int))); let inputs = nursery_new_inputs(vec![varsize, finish_int_arg0()], 53); - let (bytes, _, _, _) = codegen::build_wasm_module(&inputs).expect("varsize should lower"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("varsize should lower"); validate_wasm(&bytes); assert_eq!(nursery_top_compare_count(&bytes), 0); assert!(const_immediates(&bytes).contains(&0x22)); @@ -8868,7 +8919,7 @@ fn call_malloc_nursery_variants_lower() { fn newstr_without_a_descr_injects_the_builtin_layout() { let newstr = make_op(OpCode::Newstr, &[OpRef::const_int(3)], OpRef::ref_op(1)); let inputs = nursery_new_inputs(vec![newstr, finish_int_arg0()], 53); - let (bytes, _, _, _) = + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("Newstr without descr should inject and lower"); validate_wasm(&bytes); let immediates = const_immediates(&bytes); @@ -8939,8 +8990,7 @@ fn inline_nursery_new_keeps_the_barrier_at_the_slow_path_join() { frame: codegen::FrameGeometry::compact(5, 2, 0), ca: codegen::CaParams::default(), }; - let (bytes, _, _, _) = - codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); + let (bytes, _, _) = codegen::build_wasm_module(&inputs).expect("wasm codegen should succeed"); validate_wasm(&bytes); assert_eq!( direct_write_barrier_call_count(&bytes, WB_TARGET as i32), @@ -8949,7 +8999,7 @@ fn inline_nursery_new_keeps_the_barrier_at_the_slow_path_join() { ); let mut control = inputs; control.nursery = None; - let (bytes, _, _, _) = codegen::build_wasm_module(&control).unwrap(); + let (bytes, _, _) = codegen::build_wasm_module(&control).unwrap(); assert_eq!(direct_write_barrier_call_count(&bytes, WB_TARGET as i32), 1); } diff --git a/majit/majit-backend/src/deadframe.rs b/majit/majit-backend/src/deadframe.rs index 483f18c6a50..25c77b09ed2 100644 --- a/majit/majit-backend/src/deadframe.rs +++ b/majit/majit-backend/src/deadframe.rs @@ -540,16 +540,11 @@ impl JitFrameDeadFrame { } } - /// Present a frame that is still executing as a deadframe, without taking - /// its `jf_gcmap` over. + /// Present a frame allocated by the GC as a rooted deadframe. /// /// `llmodel.py force` casts the resolved frame to a GCREF and - /// returns it: the forced frame IS the deadframe, and it belongs to the - /// compiled run that is still on the JF shadow stack. That run pushed the - /// map before the residual call it is inside and clears it with - /// `pop_gcmap` when the call returns, so releasing it here would leave the - /// frame untraced for the rest of the call while its spilled `Ref` slots - /// are still the only reference to their objects. + /// returns it: the forced frame IS the deadframe, whether it is still + /// executing or has returned through GUARD_NOT_FORCED_2 / FINISH. pub fn borrowing( jf_gcref: GcRef, fail_descr: ExitDescr, diff --git a/majit/majit-backend/src/lib.rs b/majit/majit-backend/src/lib.rs index 2a2a743554f..53c949d3891 100644 --- a/majit/majit-backend/src/lib.rs +++ b/majit/majit-backend/src/lib.rs @@ -1985,6 +1985,15 @@ impl DeadFrame { DeadFrame::JitFrame(_) | DeadFrame::LibcJitFrame(_) => None, } } + + /// Mutable erased payload for backend-specific deadframe operations. + #[inline] + pub fn boxed_data_mut(&mut self) -> Option<&mut dyn std::any::Any> { + match self { + DeadFrame::Boxed(data) => Some(&mut **data), + DeadFrame::JitFrame(_) | DeadFrame::LibcJitFrame(_) => None, + } + } } /// `compile.py` `make_and_attach_done_descrs` + `pyjitpl.py` diff --git a/majit/majit-backend/src/libc_deadframe.rs b/majit/majit-backend/src/libc_deadframe.rs index 870d09547db..6e17c874702 100644 --- a/majit/majit-backend/src/libc_deadframe.rs +++ b/majit/majit-backend/src/libc_deadframe.rs @@ -125,6 +125,16 @@ impl LibcJitFrameDeadFrame { self.tip as usize } + #[inline] + pub fn get_savedata_ref(&self) -> GcRef { + GcRef(unsafe { (*self.tip).jf_savedata }) + } + + #[inline] + pub fn set_savedata_ref(&mut self, data: GcRef) { + unsafe { (*self.tip).jf_savedata = data.0 }; + } + /// `llmodel.py _decode_pos` — the jitframe slot logical failarg /// `index` lives in, or `None` when the descr maps it nowhere. /// diff --git a/majit/majit-gc/src/collector.rs b/majit/majit-gc/src/collector.rs index 9c37aa4421c..29a2f55ab17 100644 --- a/majit/majit-gc/src/collector.rs +++ b/majit/majit-gc/src/collector.rs @@ -1776,6 +1776,7 @@ impl MiniMarkGC { type_id, payload_size, root, + 1, needs_write_barrier, ) } @@ -1802,6 +1803,35 @@ impl MiniMarkGC { type_id, payload_size, root, + 1, + needs_write_barrier, + ) + } + } + + /// `malloc_fast` with the full `gc_push_roots(livevars)` span. The + /// common bump path touches none of the slots; only the collection tail + /// registers and later reloads them, as `postprocess_inlining` arranges for + /// RPython's expanded shadow-stack operations. + /// + /// # Safety + /// `roots` must address `root_count` contiguous mutable [`GcRef`] slots + /// until this call returns. + #[inline] + pub unsafe fn alloc_fast_with_type_roots( + &mut self, + type_id: u32, + payload_size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, + ) -> GcRef { + unsafe { + self.alloc_with_type_rooted_body::( + type_id, + payload_size, + roots, + root_count, needs_write_barrier, ) } @@ -1815,16 +1845,21 @@ impl MiniMarkGC { &mut self, type_id: u32, payload_size: usize, - root: *mut GcRef, + roots: *mut GcRef, + root_count: usize, needs_write_barrier: *mut bool, ) -> GcRef { unsafe { *needs_write_barrier = false }; #[cfg(feature = "gc_stress")] if self.stress_collect { - unsafe { self.roots.add(root) }; + for i in 0..root_count { + unsafe { self.roots.add(roots.add(i)) }; + } self.do_collect_full(); - self.roots.remove(root); + for i in (0..root_count).rev() { + self.roots.remove(unsafe { roots.add(i) }); + } } let Some(total_size) = GcHeader::SIZE.checked_add(payload_size) else { @@ -1842,7 +1877,15 @@ impl MiniMarkGC { return self.finish_bumped_nursery_object::(ptr, type_id); } } - unsafe { self.alloc_with_type_rooted_slow(type_id, total_size, root, needs_write_barrier) } + unsafe { + self.alloc_with_type_rooted_slow( + type_id, + total_size, + roots, + root_count, + needs_write_barrier, + ) + } } /// The outcomes [`alloc_with_type_rooted`] leaves out of line: a large @@ -1856,7 +1899,8 @@ impl MiniMarkGC { &mut self, type_id: u32, total_size: usize, - root: *mut GcRef, + roots: *mut GcRef, + root_count: usize, needs_write_barrier: *mut bool, ) -> GcRef { // `needs_write_barrier` stays true for the young birth as well. The @@ -1871,9 +1915,13 @@ impl MiniMarkGC { // the same way the nursery-full arm does. if total_size >= self.config.large_object_threshold { unsafe { *needs_write_barrier = true }; - unsafe { self.roots.add(root) }; + for i in 0..root_count { + unsafe { self.roots.add(roots.add(i)) }; + } let oom = self.maybe_collect_for_external_malloc(total_size); - self.roots.remove(root); + for i in (0..root_count).rev() { + self.roots.remove(unsafe { roots.add(i) }); + } if oom { return GcRef(0); } @@ -1893,11 +1941,15 @@ impl MiniMarkGC { } self.pending_reserving_size = total_size; - unsafe { self.roots.add(root) }; + for i in 0..root_count { + unsafe { self.roots.add(roots.add(i)) }; + } self.do_collect_nursery(); self.pending_reserving_size = 0; if std::mem::take(&mut self.oom_pending) { - self.roots.remove(root); + for i in (0..root_count).rev() { + self.roots.remove(unsafe { roots.add(i) }); + } return GcRef(0); } let ptr = self.nursery.alloc(total_size); @@ -1907,7 +1959,9 @@ impl MiniMarkGC { ptr }; if ptr.is_null() && Self::nursery_allocation_size(total_size) > self.nursery.size() { - self.roots.remove(root); + for i in (0..root_count).rev() { + self.roots.remove(unsafe { roots.add(i) }); + } unsafe { *needs_write_barrier = true }; return self.alloc_in_oldgen_clear(type_id, total_size); } @@ -1921,7 +1975,9 @@ impl MiniMarkGC { ptr = self.reserve_nursery_gap(total_size); } } - self.roots.remove(root); + for i in (0..root_count).rev() { + self.roots.remove(unsafe { roots.add(i) }); + } assert!( !ptr.is_null(), "collect_and_reserve could not find nursery space for a non-large object" @@ -5246,14 +5302,12 @@ impl MiniMarkGC { // custom_trace_hook parity: use custom trace function if registered. if let Some(trace_fn) = custom_trace { // A custom trace names its own slots — for a JITFRAME, `jf_gcmap` - // decides them, not the type table. When one of those slots does - // not decode as an object, the discriminating question is whether - // the *rest* of the same trace is sound: one bad slot among sound - // ones is a bad published value, while a trace whose slots are - // mostly unsound is a map that no longer describes the object. - // Defer the first undecodable slot so the whole walk completes and - // the panic can report both. - let mut deferred: Option<(usize, usize)> = None; + // decides them, not the type table. `is_nursery_object_start` is + // only a range check, so a wasm JitFrame slot that holds a scalar + // or an interior address can land inside the nursery without + // being an object start. Copying that word is the invalid-type_id + // panic; leave it alone, matching the wasm gcmap contract that a + // non-object slot is traced harmlessly. unsafe { trace_fn(obj_addr, &mut |slot_ptr: *mut GcRef| { let field_ref = *slot_ptr; @@ -5277,28 +5331,12 @@ impl MiniMarkGC { slot_ptr as usize, ); *slot_ptr = new_ref; - return; } - if deferred.is_none() { - deferred = Some((slot_ptr as usize, field_ref.0)); - } - return; } else if self.is_young_rawmalloced(field_ref.0) { self.visit_young_rawmalloced_object(field_ref.0); } }); } - if let Some((slot_addr, field)) = deferred { - let walk = self.describe_custom_trace_slots(obj_addr, trace_fn); - eprintln!("GC BUG: custom-trace slot walk for holder={obj_addr:#x}: {walk}"); - self.copy_nursery_object( - field, - "minor_custom_trace_target", - site, - obj_addr, - slot_addr, - ); - } return; } @@ -6672,6 +6710,7 @@ impl MiniMarkGC { /// /// For a JITFRAME the slot set is whatever `jf_gcmap` says, so this is the /// only way to see the map the collector actually acted on. + #[allow(dead_code)] fn describe_custom_trace_slots( &self, obj_addr: usize, @@ -8855,6 +8894,19 @@ impl GcAllocator for MiniMarkGC { unsafe { self.alloc_fast_with_type_rooted(type_id, size, root, needs_write_barrier) } } + unsafe fn alloc_fast_nursery_collecting_typed_roots( + &mut self, + type_id: u32, + size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, + ) -> GcRef { + unsafe { + self.alloc_fast_with_type_roots(type_id, size, roots, root_count, needs_write_barrier) + } + } + fn alloc_nursery_no_collect(&mut self, size: usize) -> GcRef { self.alloc_with_type_no_collect(0, size) } @@ -11286,6 +11338,39 @@ mod tests { assert_eq!(gc.roots.len(), roots_before); } + #[test] + fn rooted_collecting_alloc_forwards_every_livevar_in_span() { + let mut gc = test_gc(256); + let tid = gc.register_type(TypeInfo::simple(16)); + let mut roots = [gc.alloc_with_type(tid, 16), gc.alloc_with_type(tid, 16)]; + let old_roots = roots; + + while gc.nursery.remaining() >= GcHeader::SIZE + 16 { + let filler = gc.alloc_with_type_no_collect(tid, 16); + assert!(gc.is_in_nursery(filler.0)); + } + let roots_before = gc.roots.len(); + let mut needs_write_barrier = true; + + let parent = unsafe { + gc.alloc_fast_with_type_roots( + tid, + 16, + roots.as_mut_ptr(), + roots.len(), + &mut needs_write_barrier, + ) + }; + + assert_eq!(gc.minor_collections, 1); + assert!(roots.iter().all(|root| !gc.is_in_nursery(root.0))); + assert_ne!(roots[0], old_roots[0]); + assert_ne!(roots[1], old_roots[1]); + assert!(gc.is_in_nursery(parent.0)); + assert!(!needs_write_barrier); + assert_eq!(gc.roots.len(), roots_before); + } + #[test] fn rooted_collecting_alloc_reports_oldgen_creation_barrier() { let mut gc = test_gc(1024); diff --git a/majit/majit-gc/src/lib.rs b/majit/majit-gc/src/lib.rs index 9a365905804..e4c78ab7843 100644 --- a/majit/majit-gc/src/lib.rs +++ b/majit/majit-gc/src/lib.rs @@ -620,6 +620,35 @@ pub trait GcAllocator: Send { } } + /// The `gc_push_roots(livevars)` form of + /// [`Self::alloc_fast_nursery_collecting_typed_rooted`]. RPython's GC + /// transform publishes every live GC variable, not a distinguished single + /// one; this span is the direct ABI for allocation sites with more than one + /// native-stack livevar. + /// + /// # Safety + /// `roots` must address `root_count` contiguous mutable [`GcRef`] slots + /// which remain valid until this call returns. `needs_write_barrier` has + /// the same contract as the one-root form. + unsafe fn alloc_fast_nursery_collecting_typed_roots( + &mut self, + type_id: u32, + size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, + ) -> GcRef { + unsafe { *needs_write_barrier = true }; + for i in 0..root_count { + unsafe { self.add_root(roots.add(i)) }; + } + let result = self.alloc_nursery_typed(type_id, size); + for i in (0..root_count).rev() { + self.remove_root(unsafe { roots.add(i) }); + } + result + } + /// Allocate a fixed-size object without triggering collection. /// /// Implementations may fall back to old-gen allocation when the nursery @@ -1513,6 +1542,24 @@ impl GcAllocator for GcHandle { gc.alloc_fast_nursery_collecting_typed_rooted(type_id, size, root, needs_write_barrier) }) } + unsafe fn alloc_fast_nursery_collecting_typed_roots( + &mut self, + type_id: u32, + size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, + ) -> GcRef { + gc_sync::gc_op(|gc| unsafe { + gc.alloc_fast_nursery_collecting_typed_roots( + type_id, + size, + roots, + root_count, + needs_write_barrier, + ) + }) + } fn alloc_nursery_no_collect(&mut self, size: usize) -> GcRef { gc_sync::gc_op(|gc| gc.alloc_nursery_no_collect(size)) } @@ -2425,6 +2472,31 @@ pub unsafe fn standalone_alloc_fast_nursery_collecting_typed_rooted( }) } +/// [`standalone_alloc_fast_nursery_collecting_typed_rooted`] with the complete +/// live-root span produced by an RPython-style `gc_push_roots(livevars)`. +/// +/// # Safety +/// `roots` must address `root_count` contiguous mutable [`GcRef`] slots for the +/// duration of the call. +#[inline] +pub unsafe fn standalone_alloc_fast_nursery_collecting_typed_roots( + type_id: u32, + payload_size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, +) -> GcRef { + gc_sync::gc_op(|g| unsafe { + g.alloc_fast_nursery_collecting_typed_roots( + type_id, + payload_size, + roots, + root_count, + needs_write_barrier, + ) + }) +} + /// Process-global callback that performs a nursery allocation for the /// currently active backend. The callback returns `GcRef(0)` (i.e. /// null) on allocation failure so callers can fall back to a @@ -2809,6 +2881,65 @@ pub unsafe fn alloc_fast_nursery_collecting_typed_rooted( } } +/// Process-global root-span companion of +/// [`alloc_fast_nursery_collecting_typed_rooted`]. +pub type AllocNurseryCollectingTypedRootsFn = unsafe fn( + type_id: u32, + payload_size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, +) -> GcRef; + +global_hook!( + static ACTIVE_ALLOC_NURSERY_COLLECTING_TYPED_ROOTS: + AllocNurseryCollectingTypedRootsFn +); + +pub fn set_active_alloc_nursery_collecting_typed_roots( + hook: Option, +) { + ACTIVE_ALLOC_NURSERY_COLLECTING_TYPED_ROOTS.set(hook); +} + +/// Allocate through the active `malloc_fast` root-span allocator. +/// +/// # Safety +/// `roots` must address `root_count` contiguous mutable [`GcRef`] slots for the +/// duration of the call. +pub unsafe fn alloc_fast_nursery_collecting_typed_roots( + type_id: u32, + payload_size: usize, + roots: *mut GcRef, + root_count: usize, + needs_write_barrier: *mut bool, +) -> GcRef { + match ACTIVE_ALLOC_NURSERY_COLLECTING_TYPED_ROOTS.get() { + Some(_) if !gc_box_installed() => unsafe { + standalone_alloc_fast_nursery_collecting_typed_roots( + type_id, + payload_size, + roots, + root_count, + needs_write_barrier, + ) + }, + Some(f) => unsafe { + f( + type_id, + payload_size, + roots, + root_count, + needs_write_barrier, + ) + }, + None => { + unsafe { *needs_write_barrier = true }; + GcRef(0) + } + } +} + /// Process-global callback that runs `GcAllocator::collect_generation` on the /// active backend's GC. App-level `gc.collect(n)` reaches the live GC through /// this trampoline, carrying the generation it was given: the backends do not diff --git a/majit/majit-metainterp/src/allvirtuals.rs b/majit/majit-metainterp/src/allvirtuals.rs new file mode 100644 index 00000000000..cc6b09bb702 --- /dev/null +++ b/majit/majit-metainterp/src/allvirtuals.rs @@ -0,0 +1,183 @@ +//! `compile.py AllVirtuals` — the GC object stored in `JITFRAME.jf_savedata`. +//! +//! RPython stores the virtual cache materialized by +//! `ResumeGuardForcedDescr.handle_async_forcing` on the deadframe itself. +//! Keeping the same owner is important: a virtualizable can move in the +//! nursery, whereas the deadframe remains the object whose failing guard will +//! consume the cache. + +use std::sync::atomic::{AtomicU32, Ordering}; + +use majit_ir::GcRef; + +const UNREGISTERED: u32 = u32::MAX; +static ALL_VIRTUALS_TYPE_ID: AtomicU32 = AtomicU32::new(UNREGISTERED); + +#[repr(C, align(8))] +struct AllVirtuals { + ptr_count: usize, + int_count: usize, + length: usize, +} + +const ITEMS_OFFSET: usize = std::mem::size_of::(); + +unsafe fn all_virtuals_trace(obj_addr: usize, visit: &mut dyn FnMut(*mut GcRef)) { + let object = obj_addr as *mut AllVirtuals; + let ptr_count = unsafe { (*object).ptr_count }; + let length = unsafe { (*object).length }; + assert!( + ptr_count <= length, + "AllVirtuals pointer count exceeds cache length" + ); + let items = unsafe { (object as *mut u8).add(ITEMS_OFFSET) as *mut i64 }; + for index in 0..ptr_count { + // The cache uses i64 slots on both targets; a wasm32 GCREF occupies + // the low half, but successive references remain eight bytes apart. + visit(unsafe { items.add(index) as *mut GcRef }); + } +} + +/// `compile.py class AllVirtuals` translated as a GC struct with one trailing +/// Signed array. The prefix says which words are GCREFs; the custom tracer +/// visits exactly that prefix and leaves the integer cache unboxed. +pub fn type_info() -> majit_gc::trace::TypeInfo { + majit_gc::trace::TypeInfo::varsize_with_custom_trace( + ITEMS_OFFSET, + std::mem::size_of::(), + std::mem::offset_of!(AllVirtuals, length), + all_virtuals_trace, + ) +} + +/// Publish the type id assigned while the frontend collector is built. +pub fn set_type_id(type_id: u32) { + ALL_VIRTUALS_TYPE_ID.store(type_id, Ordering::Release); +} + +fn type_id() -> u32 { + let type_id = ALL_VIRTUALS_TYPE_ID.load(Ordering::Acquire); + assert_ne!( + type_id, UNREGISTERED, + "AllVirtuals GC type must be registered before async forcing", + ); + type_id +} + +/// Allocate and initialize the object saved by `handle_async_forcing`. +/// +/// `force_from_resumedata` returns its pointer cache in a contiguous vector. +/// Root that whole span across the collecting `malloc_fast`, exactly as the +/// GC transformer roots the live `all_virtuals` list elements around the +/// `AllVirtuals` allocation upstream. +pub fn allocate(ptrs: Vec, ints: Vec) -> GcRef { + let mut roots: Vec = ptrs.iter().map(|&value| GcRef(value as usize)).collect(); + let length = ptrs + .len() + .checked_add(ints.len()) + .expect("AllVirtuals cache length overflow"); + let payload_size = ITEMS_OFFSET + .checked_add( + length + .checked_mul(std::mem::size_of::()) + .expect("AllVirtuals payload size overflow"), + ) + .expect("AllVirtuals payload size overflow"); + let mut needs_write_barrier = false; + let object = unsafe { + majit_gc::alloc_fast_nursery_collecting_typed_roots( + type_id(), + payload_size, + roots.as_mut_ptr(), + roots.len(), + &mut needs_write_barrier, + ) + }; + assert!(!object.is_null(), "AllVirtuals allocation failed"); + // The collecting allocator unregisters `roots` before returning. Root + // the new object and the forwarded cache across initialization and the + // old-gen write barrier; the caller's `DeadFrameRefRoots` is created + // only after this function returns. Init first, then the barrier — + // `alloc_rbigint_nursery_collecting` does the same — so the card marks + // the stores rather than an empty shell. + let object_root = majit_gc::shadow_stack::OwnerRootGuard::new(object); + let mut cache_slots: Vec = roots.iter().map(|value| value.0 as i64).collect(); + let cache_root_depth = majit_gc::shadow_stack::resume_ref_roots_depth(); + if !cache_slots.is_empty() { + unsafe { + majit_gc::shadow_stack::push_resume_ref_roots(&mut cache_slots); + } + } + let object = object_root.get(); + unsafe { + let header = object.0 as *mut AllVirtuals; + (*header).ptr_count = ptrs.len(); + (*header).int_count = ints.len(); + (*header).length = length; + let items = (header as *mut u8).add(ITEMS_OFFSET) as *mut i64; + for (index, value) in cache_slots.iter().enumerate() { + *items.add(index) = *value; + } + std::ptr::copy_nonoverlapping(ints.as_ptr(), items.add(ptrs.len()), ints.len()); + } + if needs_write_barrier && !ptrs.is_empty() { + majit_gc::gc_write_barrier(object_root.get()); + } + majit_gc::shadow_stack::pop_resume_ref_roots_to(cache_root_depth); + object_root.get() +} + +/// `ResumeGuardForcedDescr.handle_fail`: reveal the cache stored in +/// `deadframe.jf_savedata` for `resume_in_blackhole`. +pub fn reveal(object: GcRef) -> Option<(Vec, Vec)> { + if object.is_null() { + return None; + } + unsafe { + let header = object.0 as *const AllVirtuals; + let ptr_count = (*header).ptr_count; + let int_count = (*header).int_count; + assert_eq!( + (*header).length, + ptr_count + int_count, + "corrupt AllVirtuals cache length", + ); + let items = (header as *const u8).add(ITEMS_OFFSET) as *const i64; + Some(( + std::slice::from_raw_parts(items, ptr_count).to_vec(), + std::slice::from_raw_parts(items.add(ptr_count), int_count).to_vec(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trace_visits_only_pointer_cache_with_i64_slot_stride() { + assert_eq!(ITEMS_OFFSET % std::mem::align_of::(), 0); + let mut storage = vec![0u64; ITEMS_OFFSET / 8 + 4]; + let object = storage.as_mut_ptr() as *mut AllVirtuals; + unsafe { + (*object).ptr_count = 2; + (*object).int_count = 2; + (*object).length = 4; + let items = (object as *mut u8).add(ITEMS_OFFSET) as *mut i64; + *items = 0x1000; + *items.add(1) = 0x2000; + *items.add(2) = 42; + *items.add(3) = -1; + let mut slots = Vec::new(); + all_virtuals_trace(object as usize, &mut |slot| { + slots.push(slot as usize); + (*slot).0 += 0x100; + }); + assert_eq!(slots, vec![items as usize, items.add(1) as usize]); + assert_eq!( + reveal(GcRef(object as usize)), + Some((vec![0x1100, 0x2100], vec![42, -1])) + ); + } + } +} diff --git a/majit/majit-metainterp/src/compile.rs b/majit/majit-metainterp/src/compile.rs index 653dedfbf66..60facd57b75 100644 --- a/majit/majit-metainterp/src/compile.rs +++ b/majit/majit-metainterp/src/compile.rs @@ -1831,6 +1831,82 @@ pub(crate) fn normalize_closing_jump_args( ops } +fn leftover_inputarg_refs( + ops: &[majit_ir::OpRc], + present: &rustc_hash::FxHashSet, +) -> Vec { + let mut leftover = Vec::new(); + let mut consider = |r: OpRef| { + if r.is_input_arg() + && !present.contains(&r.raw()) + && leftover.iter().all(|x: &OpRef| x.raw() != r.raw()) + { + leftover.push(r); + } + }; + for op in ops { + for a in op.getarglist() { + consider(a.to_opref()); + } + if let Some(fa) = op.guard_fail_args() { + for a in fa { + consider(a.to_opref()); + } + } + } + leftover +} + +/// Fit `lengths` so the items sum to `n_array_items`. Extra items land on +/// the last array field; a short count zeros the tail. `n_arrays` is +/// `vinfo.array_fields.len()` — grow a missing last slot rather than +/// invent a field the descr walk cannot emit. +fn fit_walk_lengths(lengths: &mut Vec, n_arrays: usize, n_array_items: usize) { + if n_arrays == 0 { + lengths.clear(); + return; + } + if lengths.len() < n_arrays { + lengths.resize(n_arrays, 0); + } + let current: usize = lengths.iter().sum(); + if current == n_array_items { + return; + } + if n_array_items >= current { + if let Some(last) = lengths.last_mut() { + *last += n_array_items - current; + } + return; + } + let mut remaining = n_array_items; + for len in lengths.iter_mut() { + if remaining >= *len { + remaining -= *len; + } else { + *len = remaining; + remaining = 0; + } + } +} + +/// Types of the expanded virtualizable tail, in the same order +/// `initialize_virtualizable` mints `InputArg(num_reds + i)`. +fn expanded_vable_slot_types( + vinfo: &crate::virtualizable::VirtualizableInfo, + array_lengths: &[usize], +) -> Vec { + let mut types = Vec::new(); + for field in &vinfo.static_fields { + types.push(field.field_type); + } + for (ai, array) in vinfo.array_fields.iter().enumerate() { + let len = array_lengths.get(ai).copied().unwrap_or(0); + types.extend(std::iter::repeat(array.item_type).take(len)); + } + types +} + /// `rpython/jit/metainterp/compile.py:425-461` /// `patch_new_loop_to_load_virtualizable_fields`. /// @@ -1915,6 +1991,391 @@ pub(crate) fn normalize_closing_jump_args( /// route through this helper: it would hand such an entry a prologue that /// reads a stale length. /// +/// Densify marks a leftover ListIter with this mint index so leftover-empty +/// reloads the live frame TOS (`valuestackdepth - 1`) instead of a baked +/// locals slot. Pyre's `valuestackdepth` is the absolute +/// `locals_cells_stack_w` index (starts at `n_locals + n_cells`); TOS is +/// one below it, i.e. `n_locals + n_cells + count - 1`. +pub(crate) const LISTITER_TOS_RELOAD: u32 = u32::MAX; + +/// `W_ListIterObject` `ob_type` / `w_class` word. leftover_peel_tos +/// scans the red frame for this type when TOS is not the iterator. +/// Zero (tests, pre-boot) keeps the TOS-only peel. +static LISTITER_TYPE_WORD: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Production `is_list_iter` (typed `ob_type` check). Tests leave this +/// null and use `LISTITER_TYPE_WORD`. +static LISTITER_PRED: std::sync::atomic::AtomicPtr<()> = + std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()); + +/// JIT boot: `&LIST_ITER_TYPE as *const _ as usize`. +pub fn register_listiter_type_word(word: usize) { + LISTITER_TYPE_WORD.store(word, std::sync::atomic::Ordering::Relaxed); +} + +/// JIT boot: `is_list_iter` as a C predicate. Preferred over the type word. +pub fn register_listiter_pred(f: unsafe extern "C" fn(*const u8) -> i32) { + LISTITER_PRED.store(f as *mut (), std::sync::atomic::Ordering::Relaxed); +} + +/// `execute_assembler`: the live EC top frame (`vref_referent`, never a vref). +pub fn register_leftover_scan_frame(frame: *const u8) { + LEFTOVER_SCAN_FRAME.with(|c| c.set(frame as *mut u8)); +} + +fn leftover_scan_frame() -> *mut u8 { + LEFTOVER_SCAN_FRAME.with(|c| c.get()) +} + +thread_local! { + static LEFTOVER_SCAN_FRAME: std::cell::Cell<*mut u8> = + const { std::cell::Cell::new(std::ptr::null_mut()) }; + static LEFTOVER_EMPTY_REJECT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// leftover-empty would leftover_peel_tos a non-iterator portal TOS +/// (ZipInfo). compile_loop must abort rather than install that loop. +pub fn take_leftover_empty_reject() -> bool { + LEFTOVER_EMPTY_REJECT.with(|c| c.replace(false)) +} + +fn note_leftover_empty_reject() { + LEFTOVER_EMPTY_REJECT.with(|c| c.set(true)); +} + +fn leftover_has_listiter_id() -> bool { + !LISTITER_PRED + .load(std::sync::atomic::Ordering::Relaxed) + .is_null() + || LISTITER_TYPE_WORD.load(std::sync::atomic::Ordering::Relaxed) != 0 +} + +fn leftover_ptr_is_frame(vable: *const u8, p: *const u8) -> bool { + if vable.is_null() || p.is_null() || (p as usize) & 1 != 0 { + return false; + } + let vable_is_gc = majit_gc::gc_owns_object(vable as usize); + if vable_is_gc && !majit_gc::gc_owns_object(p as usize) { + return false; + } + let vable_ty = unsafe { *(vable as *const usize) }; + let vable_class = unsafe { *(vable as *const usize).add(1) }; + let vable_tid = gc_type_id(vable as usize); + let ty = unsafe { *(p as *const usize) }; + let class = unsafe { *(p as *const usize).add(1) }; + (vable_ty != 0 && (ty == vable_ty || class == vable_ty)) + || (vable_class != 0 && (ty == vable_class || class == vable_class)) + || vable_tid.is_some_and(|tid| gc_type_id(p as usize) == Some(tid)) +} + +fn leftover_ptr_is_listiter(p: *const u8) -> bool { + if p.is_null() || (p as usize) & 1 != 0 { + return false; + } + let pred = LISTITER_PRED.load(std::sync::atomic::Ordering::Relaxed); + if !pred.is_null() { + let f: unsafe extern "C" fn(*const u8) -> i32 = unsafe { std::mem::transmute(pred) }; + return unsafe { f(p) } != 0; + } + let ty = LISTITER_TYPE_WORD.load(std::sync::atomic::Ordering::Relaxed); + if ty == 0 { + return false; + } + let obj_ty = unsafe { *(p as *const usize) }; + let obj_class = unsafe { *(p as *const usize).add(1) }; + obj_ty == ty || obj_class == ty +} + +/// TOS slots from the portal frame down through inlined callee +/// frames. Recursive `_compile` can stack several frames on one +/// portal; leftover-empty must peel until TOS is not a frame. +pub(crate) unsafe fn live_tos_for_vable( + vinfo: &crate::virtualizable::VirtualizableInfo, + vable: *const u8, +) -> Option> { + if vable.is_null() { + return None; + } + let vsd_i = vinfo + .static_fields + .iter() + .position(|f| f.name == "valuestackdepth")?; + if vinfo.array_fields.is_empty() { + let vsd = unsafe { vinfo.read_field(vable, vsd_i) as usize }; + return Some(vec![vsd.saturating_sub(1)]); + } + let vable_ty = unsafe { *(vable as *const usize) }; + let vable_class = unsafe { *(vable as *const usize).add(1) }; + let vable_tid = gc_type_id(vable as usize); + let mut ptr = vable; + let mut path = Vec::new(); + for _ in 0..10 { + let vsd = unsafe { vinfo.read_field(ptr, vsd_i) as usize }; + let alen = if vinfo.array_fields.is_empty() { + usize::MAX + } else { + unsafe { vinfo.get_array_length(ptr, 0) } + }; + // Heap vsd can sit past the minted / allocated array + // (stale token, or a different frame than the mint). + // GETARRAYITEM past that length is a SIGSEGV, not a peel. + if vsd == 0 || (alen != usize::MAX && vsd > alen) { + break; + } + let tos = vsd.saturating_sub(1); + if alen != usize::MAX && tos >= alen { + break; + } + path.push(tos); + let next = unsafe { vinfo.read_array_item(ptr, 0, tos) as usize }; + if next == 0 { + break; + } + let next_ty = unsafe { *(next as *const usize) }; + let next_class = unsafe { *(next as *const usize).add(1) }; + // Payload `ob_type` / `w_class` can disagree with the vable + // pointer if one side still names the GC header word. Cross- + // compare both words, then fall back to the GC type_id so two + // frames still peel when the header convention differs. + let word_match = (vable_ty != 0 && (next_ty == vable_ty || next_class == vable_ty)) + || (vable_class != 0 && (next_ty == vable_class || next_class == vable_class)); + let same_type = word_match || vable_tid.is_some_and(|tid| gc_type_id(next) == Some(tid)); + if !same_type { + break; + } + ptr = next as *const u8; + } + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + let alen0 = unsafe { vinfo.get_array_length(vable, 0) }; + eprintln!( + "leftover-tos path={path:?} vsd0={} alen={alen0} vable={vable:p} tid={vable_tid:?}", + unsafe { vinfo.read_field(vable, vsd_i) } + ); + } + Some(path) +} + +fn gc_type_id(addr: usize) -> Option { + if addr == 0 || !majit_gc::gc_owns_object(addr) { + return None; + } + let tid = unsafe { (*majit_gc::header::header_of(addr)).type_id() }; + (tid != 0).then_some(tid) +} + +/// Walk the loop-red frame's TOS through inlined callee frames and +/// return the live iterator. `execute_assembler` passes the live EC +/// top frame (the inlined `_compile` when `portal_frame_reg` still +/// names the caller). Offsets are baked from `VirtualizableInfo` so +/// this is a plain C call. +/// +/// If TOS is not a list iterator (a leftover-empty portal TOS can be +/// ZipInfo / Tokenizer / str after densify remaps by position), peel the +/// EC top frame published by `register_leftover_scan_frame`. Do not scan +/// other locals: that can return a live `frame`. The type word / +/// `is_list_iter` predicate is registered at JIT boot so this helper +/// stays a 7-arg CallR. +/// +/// CallR arg0 is the function pointer (not in `CallDescr.arg_types`). +/// +/// `kind == 0` is DirectPointer: the array field is `*[len][items…]` +/// (`FixedObjectArray`). `kind != 0` is EmbeddedArray: `ptr_off` +/// locates the container's data pointer. `kind` is a separate +/// argument so a `-1` sentinel cannot lose its sign across CallR. +/// Uniform-word CallR target. wasm `call_indirect` types residual +/// CallR as `(i64×n)->i64`; the typed pointer signature below is +/// `(i32, i64×6)->i32` on wasm32 and traps. Dynasm/cranelift pass +/// Refs as machine words too, so this is the CallR ABI on every +/// backend. +pub extern "C" fn leftover_peel_tos_i64( + vable: i64, + vsd_off: i64, + array_off: i64, + ptr_off: i64, + len_off: i64, + items_off: i64, + kind: i64, +) -> i64 { + unsafe { + leftover_peel_tos( + vable as *const u8, + vsd_off, + array_off, + ptr_off, + len_off, + items_off, + kind, + ) as i64 + } +} + +pub unsafe extern "C" fn leftover_peel_tos( + vable: *const u8, + vsd_off: i64, + array_off: i64, + ptr_off: i64, + len_off: i64, + items_off: i64, + kind: i64, +) -> *const u8 { + if vable.is_null() { + return vable; + } + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!( + "leftover-peel enter vable={vable:p} extra={:p}", + leftover_scan_frame() + ); + } + let vable_ty = unsafe { *(vable as *const usize) }; + let vable_class = unsafe { *(vable as *const usize).add(1) }; + let vable_tid = gc_type_id(vable as usize); + let vable_is_gc = majit_gc::gc_owns_object(vable as usize); + let is_frame = |p: usize| -> bool { + if p == 0 || p & 1 != 0 { + return false; + } + if vable_is_gc && !majit_gc::gc_owns_object(p) { + return false; + } + let ty = unsafe { *(p as *const usize) }; + let class = unsafe { *(p as *const usize).add(1) }; + (vable_ty != 0 && (ty == vable_ty || class == vable_ty)) + || (vable_class != 0 && (ty == vable_class || class == vable_class)) + || vable_tid.is_some_and(|tid| gc_type_id(p) == Some(tid)) + }; + let is_listiter = |p: usize| -> bool { + if p == 0 || p & 1 != 0 { + return false; + } + let pred = LISTITER_PRED.load(std::sync::atomic::Ordering::Relaxed); + if !pred.is_null() { + let f: unsafe extern "C" fn(*const u8) -> i32 = unsafe { std::mem::transmute(pred) }; + return unsafe { f(p as *const u8) } != 0; + } + let ty = LISTITER_TYPE_WORD.load(std::sync::atomic::Ordering::Relaxed); + if ty == 0 { + return false; + } + if vable_is_gc && !majit_gc::gc_owns_object(p) { + return false; + } + let obj_ty = unsafe { *(p as *const usize) }; + let obj_class = unsafe { *(p as *const usize).add(1) }; + obj_ty == ty || obj_class == ty + }; + let items_of = |frame: *const u8| -> Option<(*const usize, usize)> { + let container = unsafe { *(frame.add(array_off as usize) as *const *const u8) }; + if container.is_null() { + return None; + } + let alen = unsafe { *(container.add(len_off as usize) as *const usize) }; + let items = if kind == 0 { + unsafe { container.add(items_off as usize) as *const usize } + } else { + unsafe { *(container.add(ptr_off as usize) as *const *const usize) } + }; + if items.is_null() || alen == 0 { + return None; + } + Some((items, alen)) + }; + // Walk TOS only. Scanning other locals can return a live `frame` + // (or ZipInfo) and compile FOR_ITER against it. + let peel_one = |start: *const u8| -> *const u8 { + let mut ptr = start; + for _ in 0..10 { + let vsd = unsafe { *(ptr.add(vsd_off as usize) as *const usize) }; + let Some((items, alen)) = items_of(ptr) else { + break; + }; + if vsd == 0 { + break; + } + let tos = vsd.saturating_sub(1); + if tos >= alen { + break; + } + let next = unsafe { *items.add(tos) }; + if next == 0 { + break; + } + if !is_frame(next) { + if is_listiter(next) { + return next as *const u8; + } + return std::ptr::null(); + } + ptr = next as *const u8; + } + std::ptr::null() + }; + let found = peel_one(vable); + if !found.is_null() { + return found; + } + // Portal TOS is ZipInfo / Tokenizer / str: the live FOR_ITER + // iterator is on the inlined `_compile` (EC top), not this red. + let extra = leftover_scan_frame() as *const u8; + if !extra.is_null() && extra != vable { + let extra_found = peel_one(extra); + if !extra_found.is_null() { + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!( + "leftover-peel extra={extra:p} found={extra_found:p} portal_tos={found:p}" + ); + } + return extra_found; + } + } + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!( + "leftover-peel vable={vable:p} tos={found:p} extra={extra:p} ty={:#x}", + LISTITER_TYPE_WORD.load(std::sync::atomic::Ordering::Relaxed) + ); + } + std::ptr::null() +} + +/// Copy resume payload + failargs from the first body guard onto the +/// leftover_peel_tos GUARD_NONNULL. store_final_boxes already ran, so a +/// freshly minted descr has empty rd_numb and blackhole panics +/// `exit_layout.storage missing`. +fn attach_peel_guard_resume(ops: &[majit_ir::OpRc]) { + let Some(peel_guard) = ops.iter().find(|op| op.opcode == OpCode::GuardNonnull) else { + return; + }; + let Some(template) = ops.iter().find(|op| { + op.opcode.is_guard() && op.opcode != OpCode::GuardNonnull && op.getdescr().is_some() + }) else { + return; + }; + let (Some(src_d), Some(dst_d)) = (template.getdescr(), peel_guard.getdescr()) else { + return; + }; + let (Some(src_fd), Some(dst_fd)) = (src_d.as_fail_descr(), dst_d.as_fail_descr()) else { + return; + }; + if let Some(n) = src_fd.rd_numb_arc() { + dst_fd.set_rd_numb_arc(Some(n)); + } + if let Some(c) = src_fd.rd_consts_arc() { + dst_fd.set_rd_consts_arc(Some(c)); + } + if let Some(v) = src_fd.rd_virtuals_arc() { + dst_fd.set_rd_virtuals_arc(Some(v)); + } + if let Some(p) = src_fd.rd_pendingfields_arc() { + dst_fd.set_rd_pendingfields_arc(Some(p)); + } + if let Some(fa) = template.guard_fail_args() { + let types = template.get_fail_arg_types_copy(); + peel_guard.setfailargs(fa.iter().cloned().collect()); + peel_guard.set_fail_arg_types(types.clone()); + dst_fd.set_fail_arg_types(types); + } +} + /// The assertion at the baking site below is what that invariant buys in /// code: `compile.py assert i == len(inputargs)` requires the baked /// lengths to account for EXACTLY the inputargs the tracer expanded, so a @@ -1930,7 +2391,46 @@ pub fn patch_new_loop_to_load_virtualizable_fields( entry_prefix_len: usize, index_of_virtualizable: usize, constants: &mut majit_ir::ConstMap, + entry_field_oprefs: &[OpRef], + live_from_entry: &[(OpRef, u32)], + // TOS slot path: portal first, then each inlined callee frame. + live_tos: Option>, +) { + patch_new_loop_to_load_virtualizable_fields_with_vable( + ops, + inputargs, + vinfo, + vable_array_lengths, + entry_prefix_len, + index_of_virtualizable, + constants, + entry_field_oprefs, + live_from_entry, + live_tos, + std::ptr::null(), + None, + ); +} + +pub fn patch_new_loop_to_load_virtualizable_fields_with_vable( + ops: &mut Vec, + inputargs: &mut Vec, + vinfo: &crate::virtualizable::VirtualizableInfo, + vable_array_lengths: &[usize], + entry_prefix_len: usize, + index_of_virtualizable: usize, + constants: &mut majit_ir::ConstMap, + entry_field_oprefs: &[OpRef], + live_from_entry: &[(OpRef, u32)], + live_tos: Option>, + orig_vable: *const u8, + inline_vable: Option, ) { + // `orig_vable` is the compile-time heap frame leftover-empty walked + // for lengths/TOS. Portal GETFIELDs still use the runtime red + // (`inputargs[index_of_virtualizable]`). leftover iterator remaps + // (leftover_peel_tos / Getfield seq) use `inline_vable` when that + // box is the inlined `_compile`, not the portal. // `compile.py:425-461` redirects each entry inputarg at its own // `_forwarded` slot — `box.set_forwarded(extra_ops[-1])` — and `emit_op` // walks those slots through `get_box_replacement` as it copies the body. @@ -1947,19 +2447,61 @@ pub fn patch_new_loop_to_load_virtualizable_fields( // which is what makes the two constructions equivalent. use majit_ir::{Op, OpCode, OpRef, descr::ArrayFlag}; - fn set_local_forwarded(forwarding: &mut Vec>, source: OpRef, target: Operand) { - if source.is_none() || source.is_constant() { - return; + // compile.py `box.set_forwarded` is Box identity, not a raw number. + // InputArg(n) and {Int,Ref}Op(n) share `OpRef::raw()`; a single vec + // keyed by raw remaps a leftover last_instr mint onto a live pointer + // op (SIGSEGV: ldr [8, #0x18]) and the frame onto an int local. + struct LocalForwarding { + inputargs: Vec>, + ops: Vec>, + } + + impl LocalForwarding { + fn with_op_capacity(max_runtime_ref: u32) -> Self { + Self { + inputargs: Vec::new(), + ops: vec![None; (max_runtime_ref as usize).saturating_add(1)], + } + } + + fn slot_mut(&mut self, source: OpRef) -> Option<&mut Option> { + if source.is_none() || source.is_constant() { + return None; + } + let idx = source.raw() as usize; + let bank = if source.is_input_arg() { + &mut self.inputargs + } else { + &mut self.ops + }; + if idx >= bank.len() { + bank.resize(idx + 1, None); + } + Some(&mut bank[idx]) + } + + fn slot(&self, source: OpRef) -> Option<&Operand> { + if source.is_none() || source.is_constant() { + return None; + } + let idx = source.raw() as usize; + let bank = if source.is_input_arg() { + &self.inputargs + } else { + &self.ops + }; + bank.get(idx).and_then(|s| s.as_ref()) } - let idx = source.raw() as usize; - if idx >= forwarding.len() { - forwarding.resize(idx + 1, None); + } + + fn set_local_forwarded(forwarding: &mut LocalForwarding, source: OpRef, target: Operand) { + if let Some(slot) = forwarding.slot_mut(source) { + *slot = Some(target); } - forwarding[idx] = Some(target); } fn get_local_box_replacement( - forwarding: &[Option], + forwarding: &LocalForwarding, mut opref: OpRef, ) -> Option { if opref.is_none() || opref.is_constant() { @@ -1967,13 +2509,12 @@ pub fn patch_new_loop_to_load_virtualizable_fields( } let mut found = None; loop { - let idx = opref.raw() as usize; - match forwarding.get(idx) { - Some(Some(next)) => { + match forwarding.slot(opref) { + Some(next) => { opref = next.to_opref(); found = Some(next.clone()); } - _ => return found, + None => return found, } } } @@ -1985,7 +2526,7 @@ pub fn patch_new_loop_to_load_virtualizable_fields( /// RPython rewrites a LABEL arg that *is* the forwarded inputarg. fn forward_residual_args_sharing_inputarg( ops: &[majit_ir::OpRc], - forwarding: &mut Vec>, + forwarding: &mut LocalForwarding, old_opref: OpRef, target: &Operand, ) { @@ -2009,7 +2550,7 @@ pub fn patch_new_loop_to_load_virtualizable_fields( fn emit_forwarded_patch_op( extra_ops: &mut Vec, op: &Op, - forwarding: &mut Vec>, + forwarding: &mut LocalForwarding, next_opref: &mut u32, ) { let mut emitted = op.clone(); @@ -2060,9 +2601,200 @@ pub fn patch_new_loop_to_load_virtualizable_fields( index_of_virtualizable < entry_prefix_len, "virtualizable must live inside the entry prefix (pyjitpl.py:3589 index_of_virtualizable < num_red_args)" ); - if inputargs.len() <= entry_prefix_len { - // Already reduced or no virtualizable expansion in the trace. - return; + let present: rustc_hash::FxHashSet = inputargs.iter().map(|ia| ia.index).collect(); + let leftover = leftover_inputarg_refs(ops, &present); + let mut walk_lengths = vable_array_lengths.to_vec(); + let mut field_types = expanded_vable_slot_types(vinfo, &walk_lengths); + let baked_field_len = field_types.len(); + let mut expanded_len = entry_prefix_len + field_types.len(); + // `vable_array_lengths` is `vinfo.get_array_length` at some later + // read (or a finish-path re-read). The boxes + // `initialize_virtualizable` minted are `entry_field_oprefs`. A + // leftover-empty rebuild that trusts the baked length treats live + // virtualstate boxes as last_instr (binary_slice: in=24 baked=8) + // or leaves field slots as incoming NULLs (exception: in=18 + // entry=13). Prefer the mint list when it covers the statics. + if !entry_field_oprefs.is_empty() && entry_field_oprefs.len() >= vinfo.static_fields.len() { + let n_array_items = entry_field_oprefs.len() - vinfo.static_fields.len(); + fit_walk_lengths(&mut walk_lengths, vinfo.array_fields.len(), n_array_items); + field_types = expanded_vable_slot_types(vinfo, &walk_lengths); + expanded_len = entry_prefix_len + field_types.len(); + } + let mut field_raws: rustc_hash::FxHashSet = entry_field_oprefs + .iter() + .filter(|opref| opref.is_input_arg()) + .map(|opref| opref.raw()) + .collect(); + // Sequential prefix+offset ids are only the field boxes when + // `initialize_virtualizable` minted that dense tail (`has_expanded_tail`). + // The usual heap-read path mints fresh InputArgs at high ids + // (`vable_entry_oprefs`); filling the sequential range then classifies + // virtualstate body leftovers as field slots and, after rebuild, forwards + // `last_instr` onto a locals GETARRAYITEM (or a leftover local onto the + // frame). Tests that omit `entry_field_oprefs` still use the dense ids. + if field_raws.is_empty() { + for raw in entry_prefix_len as u32..expanded_len as u32 { + field_raws.insert(raw); + } + } + let leftover_fields: Vec = leftover + .iter() + .copied() + .filter(|r| field_raws.contains(&r.raw())) + .collect(); + // compile.py `box.set_forwarded` on the virtualizable red — one box, + // `virtualizable_boxes[-1]`. Every leftover Ref that is not a field is + // *not* that red: virtualstate LABEL/JUMP/failargs keep their own + // InputArgRefs, and rewriting them all onto the frame puts `last_instr` + // in a pointer slot (SIGSEGV at locals_cells_stack_w) and the frame in + // an int local (`TypeError: 'frame' < 'int'`). + // compile.py forwards only `virtualizable_boxes[-1]`. That box is + // the red at `index_of_virtualizable` (PyFrame: InputArg 0). A + // leftover range-iterator or list Ref is not the identity; remapping + // it onto the frame is `TypeError: 'frame' object is not an iterator`. + let leftover_identity: Vec = leftover + .iter() + .copied() + .filter(|r| { + r.ty() == Some(Type::Ref) + && !field_raws.contains(&r.raw()) + && r.raw() == index_of_virtualizable as u32 + }) + .collect(); + + // leftover-empty: `execute_assembler` passes only the red prefix + // (`warmstate.py`). GETFIELD the minted (or shorter present) tail. + // Growing that walk to a longer baked `get_array_length` treats live + // virtualstate boxes as last_instr (`'frame' object is not an + // iterator` in setuptools). Extra slots stay on LABEL/JUMP and must + // not join `loop.inputargs` (`compile.py:431 inputargs[:num_red_args]`). + if leftover_fields.is_empty() && leftover_identity.is_empty() { + let n_static = vinfo.static_fields.len(); + let present_fields = inputargs.len().saturating_sub(entry_prefix_len); + if present_fields == 0 { + return; + } + let minted_fields = + if !entry_field_oprefs.is_empty() && entry_field_oprefs.len() >= n_static { + entry_field_oprefs.len() + } else { + baked_field_len + }; + let walk_fields = present_fields.min(minted_fields); + if walk_fields < n_static { + return; + } + fit_walk_lengths( + &mut walk_lengths, + vinfo.array_fields.len(), + walk_fields - n_static, + ); + field_types = expanded_vable_slot_types(vinfo, &walk_lengths); + expanded_len = entry_prefix_len + field_types.len(); + let types_match = inputargs + .get(entry_prefix_len..expanded_len) + .is_some_and(|tail| tail.iter().map(|ia| ia.tp).eq(field_types.iter().copied())); + if !types_match { + inputargs.truncate(entry_prefix_len); + return; + } + } + + // `compile.py:458 assert i == len(inputargs)` requires the expanded + // list to be exactly prefix + statics + baked array items. Virtualstate + // + densify can leave a *partial* tail (some array slots still present, + // others dropped because they went virtual) or a *long* tail (identity + // leftovers still riding as inputargs). Either shape is not the + // RPython list: walking the live `vable_array_lengths` against it + // overruns or fails the equality. Rebuild to the expected shape, then + // remap leftover field/identity refs the same way the prefix-only + // reduce path already does. + if inputargs.len() != expanded_len { + // Virtualstate + densify may already have reduced `inputargs` to the + // red prefix (`start_state.renamed_inputargs`). Skip-unmodified + // store-back and short-preamble `used_boxes` leave the original + // InputArgRefs on LABEL/JUMP/failargs. RPython remaps them via + // `box.set_forwarded` because those boxes are still in + // `loop.inputargs`. Rebuild the expanded field list when a leftover + // names a field slot; a leftover Ref that is not a field is the + // virtualizable identity and rewrites onto the vable red. + if leftover_fields.is_empty() && leftover_identity.is_empty() { + if inputargs.len() > expanded_len { + inputargs.truncate(expanded_len); + } else if inputargs.len() <= entry_prefix_len { + return; + } + } + if leftover_fields.is_empty() && inputargs.len() <= entry_prefix_len { + let vable_rc = + majit_ir::InputArgRc::new(inputargs[index_of_virtualizable].fresh_value_copy()); + let vable_box = Operand::from_bound_inputarg(&vable_rc); + let max_runtime_ref = leftover_identity + .iter() + .map(|r| r.raw()) + .chain(ops.iter().flat_map(|op| { + std::iter::once(op.pos().get()) + .chain(op.getarglist_copy().into_iter().map(|b| b.to_opref())) + .chain( + op.guard_fail_args() + .into_iter() + .flatten() + .map(|b| b.to_opref()), + ) + .map(|r| { + if r.is_none() || r.is_constant() { + 0 + } else { + r.raw() + } + }) + })) + .max() + .unwrap_or(0); + let mut next_opref = max_runtime_ref + 1; + let mut forwarding = LocalForwarding::with_op_capacity(max_runtime_ref); + for &identity in &leftover_identity { + set_local_forwarded(&mut forwarding, identity, vable_box.clone()); + } + let original_ops = std::mem::take(ops); + let mut extra_ops: Vec = Vec::new(); + for op in original_ops.iter() { + emit_forwarded_patch_op(&mut extra_ops, op, &mut forwarding, &mut next_opref); + } + *ops = extra_ops; + return; + } + let mut expanded = Vec::with_capacity(expanded_len); + for i in 0..entry_prefix_len { + expanded.push( + inputargs + .iter() + .find(|ia| ia.index as usize == i) + .map(InputArg::fresh_value_copy) + .unwrap_or_else(|| InputArg::from_type(Type::Ref, i as u32)), + ); + } + for (offset, ty) in field_types.into_iter().enumerate() { + // Densify numbers surviving field slots 0..N. Those ids are + // what LABEL/JUMP/failargs still name. `vable_entry_oprefs` + // keeps the pre-densify mint (often 50+). Prefer the densified + // slot so `box.set_forwarded` hits the ops; still snap-forward + // the mint below when a leftover kept that id. + let dense_idx = entry_prefix_len + offset; + let idx = inputargs + .get(dense_idx) + .map(|ia| ia.index) + .or_else(|| { + entry_field_oprefs + .get(offset) + .copied() + .filter(|opref| opref.is_input_arg()) + .map(OpRef::raw) + }) + .unwrap_or(dense_idx as u32); + expanded.push(InputArg::from_type(ty, idx)); + } + *inputargs = expanded; } let expanded_inputargs: Vec = inputargs @@ -2107,10 +2839,10 @@ pub fn patch_new_loop_to_load_virtualizable_fields( .map(|m| m + 1) .unwrap_or(0); - let mut forwarding: Vec> = - vec![None; (max_runtime_ref as usize).saturating_add(1)]; + let mut forwarding = LocalForwarding::with_op_capacity(max_runtime_ref); let mut extra_ops: Vec = Vec::new(); let mut i = entry_prefix_len; + let mut field_bounds: Vec = Vec::new(); // compile.py:431-432 — i = jitdriver_sd.num_red_args; loop.inputargs = // inputargs[:i]. `entry_prefix_len` is that `i`, stated by the caller @@ -2146,9 +2878,18 @@ pub fn patch_new_loop_to_load_virtualizable_fields( op.pos().set(new_opref); op.setdescr(descr); let op = OpRc::new(op); - let target = Operand::from_bound_op(&op); - set_local_forwarded(&mut forwarding, old_opref, target.clone()); - forward_residual_args_sharing_inputarg(ops, &mut forwarding, old_opref, &target); + let bound = Operand::from_bound_op(&op); + set_local_forwarded(&mut forwarding, old_opref, bound.clone()); + forward_residual_args_sharing_inputarg(ops, &mut forwarding, old_opref, &bound); + field_bounds.push(bound.clone()); + if let Some(&snap) = entry_field_oprefs.get(fi) { + if snap.is_input_arg() + && snap != old_opref + && leftover_fields.iter().any(|r| r.raw() == snap.raw()) + { + set_local_forwarded(&mut forwarding, snap, bound); + } + } extra_ops.push(op); i += 1; } @@ -2166,7 +2907,7 @@ pub fn patch_new_loop_to_load_virtualizable_fields( // this trace's GREENS, so a virtualizable with other lengths keys to // a different trace and never reaches this entry. See the SOUNDNESS // INVARIANT on this function. - let array_len = vable_array_lengths.get(ai).copied().unwrap_or(0); + let array_len = walk_lengths.get(ai).copied().unwrap_or(0); assert!( i + array_len <= expanded_inputargs.len(), "array {ai} length {array_len} would overrun inputargs (i={i}, len={})", @@ -2297,9 +3038,18 @@ pub fn patch_new_loop_to_load_virtualizable_fields( elem_op.pos().set(new_opref); elem_op.setdescr(item_descr.clone()); let elem_op = OpRc::new(elem_op); - let target = Operand::from_bound_op(&elem_op); - set_local_forwarded(&mut forwarding, old_opref, target.clone()); - forward_residual_args_sharing_inputarg(ops, &mut forwarding, old_opref, &target); + let bound = Operand::from_bound_op(&elem_op); + set_local_forwarded(&mut forwarding, old_opref, bound.clone()); + forward_residual_args_sharing_inputarg(ops, &mut forwarding, old_opref, &bound); + field_bounds.push(bound.clone()); + if let Some(&snap) = entry_field_oprefs.get(i - entry_prefix_len) { + if snap.is_input_arg() + && snap != old_opref + && leftover_fields.iter().any(|r| r.raw() == snap.raw()) + { + set_local_forwarded(&mut forwarding, snap, bound); + } + } extra_ops.push(elem_op); i += 1; } @@ -2317,51 +3067,413 @@ pub fn patch_new_loop_to_load_virtualizable_fields( expanded_inputargs.len() ); - // compile.py — emit_op walks the existing ops re-emitting - // each one with `get_box_replacement` applied to args + fail_args. - let original_ops = std::mem::take(ops); - for op in original_ops.iter() { - emit_forwarded_patch_op(&mut extra_ops, op, &mut forwarding, &mut next_opref); - } - *ops = extra_ops; -} - -/// RPython dependency.py requires GUARD_(NO_)OVERFLOW to be scheduled only -/// when there is a live preceding INT_*_OVF operation to consume. -/// intbounds.py:231-242: optimizer raises InvalidLoop for stray overflow -/// guards. This function is a post-optimization safety net: if any stray -/// guard survived, strip it to prevent backend panic. -pub(crate) fn strip_stray_overflow_guards(ops: Vec) -> Vec { - use majit_ir::OpCode; - - let mut pending_ovf = false; - let mut result = Vec::with_capacity(ops.len()); - for op in ops { - match op.opcode { - OpCode::IntAddOvf | OpCode::IntSubOvf | OpCode::IntMulOvf => { - pending_ovf = true; - result.push(op); - } - OpCode::GuardNoOverflow | OpCode::GuardOverflow => { - if pending_ovf { - result.push(op); + // compile.py `box.set_forwarded` on the virtualizable red: a leftover + // InputArgRef that is not a field snapshot is the identity box + // virtualstate left on LABEL/JUMP/failargs after densify dropped it. + for &identity in &leftover_identity { + set_local_forwarded(&mut forwarding, identity, vable_box.clone()); + } + + // compile.py walks field boxes by Box identity. The positional + // `InputArg(prefix+i)` remap above is the densified fallback. + // A mint-identity bind (renamed[p] == mint[j]) must win: GET_ITER + // of that slot reloads field j, not the valuestack slot at p. + // Portal TOS is an index into *this* vable's array. leftover_peel_tos + // reads the live heap (alen=38 on `_compile`) so a heap vsd past the + // mint walk (31/29 leftover-empty: path=[28] on a 25-slot mint) must + // still emit the peel. GETARRAYITEM below stays on `walk_lengths`. + // A previous clamp dropped that peel and remapped FOR_ITER onto + // locals[22] (ZipInfo / `_compile.p`). + + // ListIter leftovers use LISTITER_TOS_RELOAD: the live TOS slot + // `valuestackdepth - 1`, not a baked local. Also pick up leftover + // Getfield(seq) receivers densify missed (descr name after opt). + let mut tos_sources: Vec = live_from_entry + .iter() + .filter(|(_, j)| *j == LISTITER_TOS_RELOAD) + .map(|(s, _)| *s) + .collect(); + let mut mint_seq_frame = false; + for op in ops.iter() { + if op.opcode != OpCode::GetfieldGcR { + continue; + } + if !op.getdescr().is_some_and(|d| { + d.as_field_descr() + .is_some_and(crate::history::is_list_iter_seq_field) + }) { + continue; + } + if let Some(src) = op.getarglist().first().map(|a| a.to_opref()) { + // Mint-field ListIter.seq is reloaded by GETFIELD when the + // slot is a listiter. Leftover extras past the mint, or a + // mint slot that is a frame, must not stay on GETFIELD + // (`'frame' object is not an iterator`). + if src.is_input_arg() && src.ty() == Some(Type::Ref) && !tos_sources.contains(&src) { + if src.raw() >= expanded_len as u32 { + tos_sources.push(src); + } else if leftover_has_listiter_id() + && !orig_vable.is_null() + && src.raw() >= entry_prefix_len as u32 + && (src.raw() as usize) < expanded_len + { + let idx = src.raw() as usize - entry_prefix_len; + let n_static = vinfo.static_fields.len(); + let slot = if idx < n_static { + unsafe { vinfo.read_field(orig_vable, idx) as *const u8 } + } else if !vinfo.array_fields.is_empty() { + unsafe { vinfo.read_array_item(orig_vable, 0, idx - n_static) as *const u8 } + } else { + std::ptr::null() + }; + if !leftover_ptr_is_listiter(slot) { + tos_sources.push(src); + mint_seq_frame = true; + } } - // else: stray guard — strip it (intbounds.py:231 InvalidLoop - // should have caught it; this is a safety net). - pending_ovf = false; - } - OpCode::Label | OpCode::Jump | OpCode::Finish => { - pending_ovf = false; - result.push(op); - } - _ => { - result.push(op); } } } - result -} - + // Leftover extras past the mint cannot stay positional. Mint-field + // TOS_RELOAD can fall back to GETFIELD when peel preview is not a + // listiter (range-for). Aborting those traces SNAPDIFF'd unrelated + // fixtures. + let listiter_leftover = tos_sources.iter().any(|s| s.raw() >= expanded_len as u32); + // ForIterNext residual leftover (no Getfield seq). Bind it to the + // peeled TOS only when leftover-empty would otherwise GETFIELD a + // portal slot that is not the iterator. Leftover numbering is the + // portal field space (`InputArg(28)` = field 26). After a peel the + // portal TOS *is* the callee frame — leftover at that field must + // follow the inner TOS, not stay on GETARRAYITEM(portal). A leftover + // already at the unpeeled TOS field (range for-loops) stays + // positional; remapping every ForIterNext leftover GC-panicked. + if let Some(path) = live_tos.as_ref() { + if let Some(&portal_tos) = path.first() { + let n_static = vinfo.static_fields.len(); + let portal_tos_inputarg = (entry_prefix_len + n_static + portal_tos) as u32; + let peeled = path.len() > 1; + for op in ops.iter() { + if !op.opcode.is_call() { + continue; + } + let is_foriter = op.getdescr().is_some_and(|d| { + d.as_call_descr().is_some_and(|cd| { + cd.get_extra_info().runtime_helper + == majit_ir::RuntimeHelperKind::ForIterNext + }) + }); + if !is_foriter { + continue; + } + for arg in op.getarglist() { + let src = arg.to_opref(); + // Densified leftover below the portal TOS, or *at* + // the portal TOS after a peel (that slot is a frame). + // High-id split leftovers stay off this bind. + // Call args include ConstInt helper targets; do not + // call `raw()` until `is_input_arg` is true. + if !src.is_input_arg() || src.ty() != Some(Type::Ref) { + continue; + } + let below_portal = src.raw() < portal_tos_inputarg; + let at_peeled_frame = peeled && src.raw() == portal_tos_inputarg; + let at_unpeeled_tos = !peeled && src.raw() == portal_tos_inputarg; + if src.raw() >= entry_prefix_len as u32 + && (below_portal || at_peeled_frame) + && !tos_sources.contains(&src) + { + tos_sources.push(src); + } + if leftover_has_listiter_id() + && !orig_vable.is_null() + && src.raw() >= entry_prefix_len as u32 + && (src.raw() as usize) < expanded_len + && (below_portal || at_peeled_frame || at_unpeeled_tos) + { + let idx = src.raw() as usize - entry_prefix_len; + let n_static = vinfo.static_fields.len(); + let slot = if idx < n_static { + unsafe { vinfo.read_field(orig_vable, idx) as *const u8 } + } else if !vinfo.array_fields.is_empty() { + unsafe { + vinfo.read_array_item(orig_vable, 0, idx - n_static) as *const u8 + } + } else { + std::ptr::null() + }; + if leftover_ptr_is_frame(orig_vable, slot) { + mint_seq_frame = true; + } + } + } + } + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + let leftover_raws: Vec = leftover.iter().map(|r| r.raw()).collect(); + eprintln!( + "leftover-empty expanded={} baked={} entry={} prefix={} n_static={} \ + path={path:?} portal_tos_ia={portal_tos_inputarg} peeled={peeled} \ + leftover={leftover_raws:?} tos_src={:?} inline_vable={inline_vable:?}", + expanded_inputargs.len(), + baked_field_len, + entry_field_oprefs.len(), + entry_prefix_len, + n_static, + tos_sources.iter().map(|r| r.raw()).collect::>(), + ); + } + } + } + let peel_vable_prologue = match inline_vable { + Some(r) if r.is_input_arg() && (r.raw() as usize) < entry_prefix_len => Some( + Operand::from_bound_inputarg(&expanded_inputargs[r.raw() as usize]), + ), + Some(r) if r.is_input_arg() => Some(Operand::from_opref(r)), + Some(_) => None, + None => Some(vable_box.clone()), + }; + let emit_peel = |extra_ops: &mut Vec, + next_opref: &mut u32, + peel_vable: Operand| + -> Option { + let vsd_f = vinfo + .static_fields + .iter() + .find(|f| f.name == "valuestackdepth")?; + let arr = vinfo.array_fields.first()?; + let (ptr_off, kind) = match arr.storage { + crate::virtualizable::VableArrayStorage::EmbeddedArray { ptr_offset } => { + (ptr_offset as i64, 1_i64) + } + _ => (0, 0_i64), + }; + let effect = majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CannotRaise, + majit_ir::OopSpecIndex::None, + ); + let descr = majit_ir::descr::make_call_descr( + vec![ + Type::Ref, + Type::Int, + Type::Int, + Type::Int, + Type::Int, + Type::Int, + Type::Int, + ], + Type::Ref, + effect, + ); + let peel_fn = if cfg!(target_arch = "wasm32") { + leftover_peel_tos_i64 as usize as i64 + } else { + leftover_peel_tos as usize as i64 + }; + let fn_op = Operand::from_opref(OpRef::const_int(peel_fn)); + let call_opref = OpRef::ref_op(*next_opref); + *next_opref += 1; + let mut call = Op::new( + OpCode::CallR, + &[ + fn_op, + peel_vable, + Operand::from_opref(OpRef::const_int(vsd_f.offset as i64)), + Operand::from_opref(OpRef::const_int(arr.field_offset as i64)), + Operand::from_opref(OpRef::const_int(ptr_off)), + Operand::from_opref(OpRef::const_int(arr.length_offset as i64)), + Operand::from_opref(OpRef::const_int(arr.items_offset as i64)), + Operand::from_opref(OpRef::const_int(kind)), + ], + ); + call.pos().set(call_opref); + call.setdescr(descr); + let call = OpRc::new(call); + extra_ops.push(call.clone()); + let bound = Operand::from_bound_op(&call); + // leftover_peel_tos returns null when the live TOS is not a + // listiter. FOR_ITER on null is a SIGSEGV; fail the guard and + // blackhole from the red prefix instead. + let fail_types: Vec = (0..entry_prefix_len) + .map(|i| expanded_inputargs[i].tp) + .collect(); + let failargs: smallvec::SmallVec<[Operand; 8]> = (0..entry_prefix_len) + .map(|i| Operand::from_bound_inputarg(&expanded_inputargs[i])) + .collect(); + let mut guard = Op::new(OpCode::GuardNonnull, std::slice::from_ref(&bound)); + guard.pos().set(OpRef::void_op(*next_opref)); + *next_opref += 1; + guard.setdescr(make_fail_descr_typed(fail_types.clone())); + guard.setfailargs(failargs); + guard.set_fail_arg_types(fail_types); + extra_ops.push(OpRc::new(guard)); + Some(bound) + }; + let mut peel_emitted = false; + let mint_tos_is_frame = leftover_has_listiter_id() + && !orig_vable.is_null() + && live_from_entry.iter().any(|(source, mint_index)| { + if *mint_index != LISTITER_TOS_RELOAD { + return false; + } + if source.raw() < entry_prefix_len as u32 || (source.raw() as usize) >= expanded_len { + return false; + } + let idx = source.raw() as usize - entry_prefix_len; + let n_static = vinfo.static_fields.len(); + let slot = if idx < n_static { + unsafe { vinfo.read_field(orig_vable, idx) as *const u8 } + } else if !vinfo.array_fields.is_empty() { + unsafe { vinfo.read_array_item(orig_vable, 0, idx - n_static) as *const u8 } + } else { + std::ptr::null() + }; + leftover_ptr_is_frame(orig_vable, slot) + }); + // A mint TOS_RELOAD whose slot is a frame is leftover-empty + // FOR_ITER (`'frame' object is not an iterator`). leftover_peel_tos + // may find a listiter at compile time (scan frame) and emit a peel, + // but a runtime null deopts through the FOR_ITER failargs — still + // the leftover frame — and TypeErrors. Refuse to compile. + if mint_tos_is_frame || mint_seq_frame { + tos_sources.clear(); + note_leftover_empty_reject(); + } + if leftover_has_listiter_id() && !tos_sources.is_empty() && !orig_vable.is_null() { + let vsd_f = vinfo + .static_fields + .iter() + .find(|f| f.name == "valuestackdepth"); + let arr = vinfo.array_fields.first(); + if let (Some(vsd_f), Some(arr)) = (vsd_f, arr) { + let (ptr_off, kind) = match arr.storage { + crate::virtualizable::VableArrayStorage::EmbeddedArray { ptr_offset } => { + (ptr_offset as i64, 1_i64) + } + _ => (0, 0_i64), + }; + let preview = unsafe { + leftover_peel_tos( + orig_vable, + vsd_f.offset as i64, + arr.field_offset as i64, + ptr_off, + arr.length_offset as i64, + arr.items_offset as i64, + kind, + ) + }; + // Production never emits leftover_peel_tos: a runtime null + // deopts through FOR_ITER failargs that still name the + // leftover frame (`'frame' object is not an iterator`). + // Tests leave LISTITER_TYPE_WORD unset so peel still emits. + // A mint TOS that *is* a listiter must stay on GETFIELD — + // rejecting that aborts every leftover-empty `for x in xs`. + tos_sources.clear(); + if listiter_leftover { + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!( + "leftover-empty reject TOS={preview:p} orig={orig_vable:p} \ + extras={listiter_leftover} mint_frame={mint_tos_is_frame}" + ); + } + note_leftover_empty_reject(); + } + } + } + if !tos_sources.is_empty() { + if let Some(peel_vable) = peel_vable_prologue { + if let Some(bound) = emit_peel(&mut extra_ops, &mut next_opref, peel_vable) { + for &source in &tos_sources { + set_local_forwarded(&mut forwarding, source, bound.clone()); + } + peel_emitted = true; + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!( + "leftover-empty peel_emitted tos_src={:?}", + tos_sources.iter().map(|r| r.raw()).collect::>() + ); + } + } + } + } + for &(source, mint_index) in live_from_entry { + if mint_index == LISTITER_TOS_RELOAD { + // Mint-field ListIter: densify stamped TOS_RELOAD, but the + // slot is a GETFIELD/GETARRAYITEM we already emitted. + if source.raw() >= entry_prefix_len as u32 && (source.raw() as usize) < expanded_len { + let idx = source.raw() as usize - entry_prefix_len; + if let Some(bound) = field_bounds.get(idx) { + set_local_forwarded(&mut forwarding, source, bound.clone()); + } + } + continue; + } + if tos_sources.contains(&source) { + continue; + } + if let Some(bound) = field_bounds.get(mint_index as usize) { + set_local_forwarded(&mut forwarding, source, bound.clone()); + } + } + + // compile.py — emit_op walks the existing ops re-emitting + // each one with `get_box_replacement` applied to args + fail_args. + let original_ops = std::mem::take(ops); + for op in original_ops.iter() { + emit_forwarded_patch_op(&mut extra_ops, op, &mut forwarding, &mut next_opref); + if !peel_emitted && !tos_sources.is_empty() && inline_vable == Some(op.pos().get()) { + if let Some(emitted) = extra_ops.last().cloned() { + let peel_vable = Operand::from_bound_op(&emitted); + if let Some(bound) = emit_peel(&mut extra_ops, &mut next_opref, peel_vable) { + for &source in &tos_sources { + set_local_forwarded(&mut forwarding, source, bound.clone()); + } + peel_emitted = true; + } + } + } + } + if peel_emitted { + attach_peel_guard_resume(&extra_ops); + } + *ops = extra_ops; +} + +/// RPython dependency.py requires GUARD_(NO_)OVERFLOW to be scheduled only +/// when there is a live preceding INT_*_OVF operation to consume. +/// intbounds.py:231-242: optimizer raises InvalidLoop for stray overflow +/// guards. This function is a post-optimization safety net: if any stray +/// guard survived, strip it to prevent backend panic. +pub(crate) fn strip_stray_overflow_guards(ops: Vec) -> Vec { + use majit_ir::OpCode; + + let mut pending_ovf = false; + let mut result = Vec::with_capacity(ops.len()); + for op in ops { + match op.opcode { + OpCode::IntAddOvf | OpCode::IntSubOvf | OpCode::IntMulOvf => { + pending_ovf = true; + result.push(op); + } + OpCode::GuardNoOverflow | OpCode::GuardOverflow => { + if pending_ovf { + result.push(op); + } + // else: stray guard — strip it (intbounds.py:231 InvalidLoop + // should have caught it; this is a safety net). + pending_ovf = false; + } + OpCode::Label | OpCode::Jump | OpCode::Finish => { + pending_ovf = false; + result.push(op); + } + _ => { + result.push(op); + } + } + } + result +} + pub(crate) fn enrich_guard_resume_layouts_for_trace( resume_layouts: &mut indexmap::IndexMap, exit_layouts: &mut crate::FxIndexMap, @@ -2738,9 +3850,31 @@ pub fn compile_tmp_callback( mod tests { use super::*; use crate::compile::make_fail_descr_with_index; - use crate::history::test_support::rooted_inputarg_operand; + use crate::history::test_support::{rooted_inputarg_operand, rooted_resop_operand}; use crate::resume::{ResumeDataLoopMemo, SimpleBoxEnv, Snapshot, SnapshotFrame}; - use majit_ir::{ArrayFlag, Op, OpCode, OpRef}; + use majit_ir::{ArrayFlag, Op, OpCode, OpRc, OpRef}; + + fn leftover_peel_index(ops: &[OpRc]) -> usize { + let i = ops + .iter() + .position(|op| op.opcode == OpCode::CallR && op.num_args() == 8) + .expect("leftover_peel_tos"); + let peel = &ops[i]; + let guard = ops + .get(i + 1) + .expect("GUARD_NONNULL after leftover_peel_tos"); + assert_eq!( + guard.opcode, + OpCode::GuardNonnull, + "leftover_peel_tos must be followed by GUARD_NONNULL" + ); + assert_eq!( + guard.arg(0).to_opref(), + peel.pos().get(), + "GUARD_NONNULL must test leftover_peel_tos" + ); + i + } /// `normalize_closing_jump_args` repairs a JUMP slot from the LABEL slot /// at the same index, which only names the same live value while the JUMP @@ -2997,6 +4131,9 @@ mod tests { 1, 0, &mut constants, + &[], + &[], + None, ); assert_eq!(inputargs, vec![InputArg::new_ref(0)]); @@ -3038,156 +4175,2097 @@ mod tests { } #[test] - fn test_patch_new_loop_reads_embedded_array_items_from_backing_storage() { + fn test_patch_new_loop_reloads_leftover_inputargs_after_virtualstate_reduce() { let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); - vinfo.add_embedded_array_field( - "locals_cells_stack_w", - Type::Ref, - 8, - 0, - 8, - 0, - majit_ir::descr::make_array_descr(0, 8, Type::Ref), - ); + vinfo.add_field("obj", Type::Ref, 8); vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); - let mut ops = vec![Op::new( + // Densify already dropped the expanded field, but skip-unmodified + // store-back left the snapshot InputArgRef on LABEL/JUMP/failargs. + let label = Op::new( OpCode::Label, &[ rooted_inputarg_operand(Type::Ref, 0), rooted_inputarg_operand(Type::Ref, 1), - rooted_inputarg_operand(Type::Ref, 2), ], - )]; - let mut inputargs = vec![ - InputArg::new_ref(0), - InputArg::new_ref(1), - InputArg::new_ref(2), - ]; + ); + let mut guard = Op::new(OpCode::GuardTrue, &[rooted_inputarg_operand(Type::Int, 0)]); + guard.setfailargs(smallvec::smallvec![ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + ]); + let jump = Op::new( + OpCode::Jump, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + ], + ); + let mut ops: Vec = vec![label, guard, jump] + .into_iter() + .map(OpRc::new) + .collect(); + let mut inputargs = vec![InputArg::new_ref(0)]; let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); - let mut ops: Vec = ops.into_iter().map(OpRc::new).collect(); patch_new_loop_to_load_virtualizable_fields( &mut ops, &mut inputargs, &vinfo, - &[2], + &[], 1, 0, &mut constants, + &[], + &[], + None, ); assert_eq!(inputargs, vec![InputArg::new_ref(0)]); - assert_eq!(ops.len(), 5); assert_eq!(ops[0].opcode, OpCode::GetfieldGcR); - assert_eq!(ops[1].opcode, OpCode::GetfieldGcI); + let reloaded = ops[0].pos().get(); assert_eq!( ops[1] .getarglist() .iter() .map(|a| a.to_opref()) .collect::>(), - vec![ops[0].pos().get()] + vec![OpRef::input_arg_ref(0), reloaded] ); - assert_eq!(ops[2].opcode, OpCode::GetarrayitemRawR); - assert_eq!(ops[2].arg(0).to_opref(), ops[1].pos().get()); - assert_eq!(ops[3].opcode, OpCode::GetarrayitemRawR); - assert_eq!(ops[3].arg(0).to_opref(), ops[1].pos().get()); - assert_eq!(ops[4].opcode, OpCode::Label); assert_eq!( - ops[4] + ops[2] + .guard_fail_args() + .unwrap() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![OpRef::input_arg_ref(0), reloaded] + ); + assert_eq!( + ops[3] .getarglist() .iter() .map(|a| a.to_opref()) .collect::>(), - vec![ - OpRef::input_arg_ref(0), - ops[2].pos().get(), - ops[3].pos().get() - ] + vec![OpRef::input_arg_ref(0), reloaded] ); } - /// Residual body-LABEL `RefOp`s that still forward to an expanded - /// inputarg are the same Box as `loop.inputargs[i]`. `emit_op` must - /// rewrite them to the GETARRAYITEM, not leave a producerless hole. #[test] - fn test_patch_new_loop_rewrites_residual_label_refop_forwarded_to_inputarg() { - use crate::history::test_support::{rooted_inputarg_operand, rooted_resop_operand}; - + fn test_patch_new_loop_leaves_non_identity_leftover_ref() { + // InputArg 98 is a virtualstate body Ref, not virtualizable_boxes[-1] + // (the prefix red at index_of_virtualizable). compile.py does not + // forward it onto the frame. let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); - vinfo.add_embedded_array_field( - "locals_cells_stack_w", - Type::Ref, - 8, - 0, - 8, - 0, - majit_ir::descr::make_array_descr(0, 8, Type::Ref), - ); vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); - let slot = rooted_inputarg_operand(Type::Ref, 1); - let reminted = rooted_resop_operand(Type::Ref, 85); - reminted.set_forwarded_inputarg( - &slot - .bound_inputarg() - .expect("rooted inputarg must carry its InputArgRc"), - ); - - let mut ops: Vec = vec![OpRc::new(Op::new( + let label = Op::new( OpCode::Label, - &[rooted_inputarg_operand(Type::Ref, 0), reminted], - ))]; - let mut inputargs = vec![InputArg::new_ref(0), InputArg::new_ref(1)]; + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 98), + ], + ); + let jump = Op::new( + OpCode::Jump, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 98), + ], + ); + let mut ops: Vec = vec![label, jump].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![InputArg::new_ref(0)]; let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); patch_new_loop_to_load_virtualizable_fields( &mut ops, &mut inputargs, &vinfo, - &[1], + &[], 1, 0, &mut constants, + &[], + &[], + None, ); assert_eq!(inputargs, vec![InputArg::new_ref(0)]); - let label = ops.iter().find(|op| op.opcode == OpCode::Label).unwrap(); - let getitem = ops - .iter() - .find(|op| op.opcode == OpCode::GetarrayitemRawR) - .unwrap(); assert_eq!( - label + ops[0] .getarglist() .iter() .map(|a| a.to_opref()) .collect::>(), - vec![OpRef::input_arg_ref(0), getitem.pos().get()] + vec![OpRef::input_arg_ref(0), OpRef::input_arg_ref(98)] + ); + assert_eq!( + ops[1] + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![OpRef::input_arg_ref(0), OpRef::input_arg_ref(98)] ); } - /// The entry prefix is the caller's number, not `num_red_args`. - /// - /// A flat entry contract puts the virtualizable at some interior slot with - /// other live entry values around it — here one int scalar at slot 0, the - /// virtualizable at slot 1, and the array elements following from slot 2. - /// `compile.py:431-432` must truncate at 2 and start reconstructing there; - /// a helper that re-derived the point from a red count (this driver - /// declares one red, the whole state) would cut at 1 and reinterpret the - /// virtualizable itself as the first array element. #[test] - fn test_patch_new_loop_truncates_at_the_callers_entry_prefix_not_a_red_count() { + fn test_patch_new_loop_does_not_rewrite_multiple_leftover_refs_as_identity() { + // Two leftover Refs after densify are virtualstate body args, not + // two copies of the vable red. compile.py forwards only + // `virtualizable_boxes[-1]`. let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); - vinfo.add_embedded_array_field( - "regs", - Type::Int, - 8, - 0, - 8, - 0, - majit_ir::descr::make_array_descr(0, 8, Type::Int), + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 98), + rooted_inputarg_operand(Type::Ref, 99), + ], + ); + let jump = Op::new( + OpCode::Jump, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 98), + rooted_inputarg_operand(Type::Ref, 99), + ], + ); + let mut ops: Vec = vec![label, jump].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![InputArg::new_ref(0)]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[], + 1, + 0, + &mut constants, + &[], + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert_eq!( + ops[0] + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![ + OpRef::input_arg_ref(0), + OpRef::input_arg_ref(98), + OpRef::input_arg_ref(99), + ] + ); + assert_eq!( + ops[1] + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![ + OpRef::input_arg_ref(0), + OpRef::input_arg_ref(98), + OpRef::input_arg_ref(99), + ] + ); + } + + #[test] + fn test_patch_new_loop_forwards_densified_field_slots_not_only_entry_mints() { + // Densify left last_instr as InputArg(2) on LABEL. The mint + // `vable_entry_oprefs` still names InputArg(50). Rebuild must + // forward 2 — the id the ops still use — or the backend treats + // InputArg(2) as a missing entry register (last_instr-as-pointer). + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("obj", Type::Ref, 16); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 24, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(32)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Int, 2), + rooted_inputarg_operand(Type::Ref, 3), + rooted_inputarg_operand(Type::Int, 50), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_int(2), + InputArg::new_ref(3), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_int(50), OpRef::input_arg_ref(51)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[2], + 1, + 0, + &mut constants, + &entry_mints, + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + let label_args: Vec = ops + .iter() + .find(|op| op.opcode == OpCode::Label) + .expect("label") + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect(); + assert!( + !label_args.iter().any(|r| r.is_input_arg() && r.raw() == 2), + "densified last_instr InputArg(2) must be forwarded to GETFIELD, got {label_args:?}" + ); + assert!( + ops.iter().any(|op| op.opcode == OpCode::GetfieldGcI), + "must emit GETFIELD_GC_I for last_instr" + ); + } + + #[test] + fn test_patch_new_loop_inputarg_forward_does_not_alias_op_raw() { + // InputArg(50) and RefOp(50) share OpRef::raw(). Forwarding the + // leftover last_instr mint must not rewrite a live pointer op. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Int, 50), + ], + ); + let get = Op::new(OpCode::GetfieldGcR, &[rooted_resop_operand(Type::Ref, 50)]); + let mut ops: Vec = vec![label, get].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![InputArg::new_ref(0)]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_int(50)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[], + 1, + 0, + &mut constants, + &entry_mints, + &[], + None, + ); + + let get_args: Vec = ops + .iter() + .find(|op| op.opcode == OpCode::GetfieldGcR) + .expect("getfield") + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect(); + assert_eq!( + get_args, + vec![OpRef::ref_op(50)], + "RefOp(50) must stay a pointer; InputArg(50) last_instr must not steal its raw" + ); + } + + #[test] + fn test_patch_new_loop_rebuilds_partially_reduced_array_tail() { + // Virtualstate kept a prefix of the array InputArgs (6 of 8) and + // dropped the rest. The live vable still reports length 8, so a + // walk that trusts `vable_array_lengths` against the partial list + // overruns (`i + 8 > 8` after the red + 4 statics, or here + // red + 2 kept items). Rebuild to the baked shape first. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("obj", Type::Ref, 8); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 16, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(24)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 2), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_ref(2), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[4], + 1, + 0, + &mut constants, + &[], + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert!( + ops.iter() + .filter(|op| op.opcode == OpCode::GetarrayitemRawR) + .count() + == 1, + "a short leftover-empty tail must GETARRAYITEM only the present slots" + ); + } + + #[test] + fn test_patch_new_loop_drops_leftover_empty_extras_from_entry() { + // leftover-empty, inputargs = prefix + minted fields + 2 extras. + // compile.py `loop.inputargs = inputargs[:num_red_args]`; extras + // stay on LABEL as leftover InputArgs and must not join the + // assembler entry (`execute_assembler` passes only the reds). + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("obj", Type::Ref, 8); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 2), + rooted_inputarg_operand(Type::Int, 3), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_ref(2), + InputArg::new_int(3), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_ref(1)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[], + 1, + 0, + &mut constants, + &entry_mints, + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert!( + ops.iter().any(|op| op.opcode == OpCode::GetfieldGcR), + "must GETFIELD the minted obj slot" + ); + let label_args: Vec = ops + .iter() + .find(|op| op.opcode == OpCode::Label) + .expect("label") + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect(); + assert_eq!(label_args.len(), 4); + assert_eq!(label_args[0], OpRef::input_arg_ref(0)); + assert!( + !label_args[1].is_input_arg(), + "minted obj must be a GETFIELD result, got {label_args:?}" + ); + assert_eq!(label_args[2], OpRef::input_arg_ref(2)); + assert_eq!(label_args[3], OpRef::input_arg_int(3)); + } + + #[test] + fn test_patch_new_loop_does_not_getfield_past_the_mint() { + // leftover-empty. Entry minted last_instr + 1 item; the live tail + // and baked length are last_instr + 3 items. Growing the walk to + // the bake treats the extra live boxes as array items (frame-as + // iterator). GETARRAYITEM only the mint; extras stay on LABEL. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 16, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(24)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Int, 1), + rooted_inputarg_operand(Type::Ref, 2), + rooted_inputarg_operand(Type::Ref, 3), + rooted_inputarg_operand(Type::Ref, 4), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_int(1), + InputArg::new_ref(2), + InputArg::new_ref(3), + InputArg::new_ref(4), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_int(1), OpRef::input_arg_ref(2)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[3], + 1, + 0, + &mut constants, + &entry_mints, + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert_eq!( + ops.iter() + .filter(|op| op.opcode == OpCode::GetarrayitemRawR) + .count(), + 1, + "must GETARRAYITEM only the minted array item, not the baked tail" + ); + let label_args: Vec = ops + .iter() + .find(|op| op.opcode == OpCode::Label) + .expect("label") + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect(); + assert_eq!(label_args[3], OpRef::input_arg_ref(3)); + assert_eq!(label_args[4], OpRef::input_arg_ref(4)); + } + + #[test] + fn test_patch_new_loop_binds_live_leftover_to_entry_slot_getfield() { + // Densify split a body-LABEL list/pattern onto InputArg(99) from + // entry slot 1. leftover-empty GETFIELDs that slot and forwards + // the live leftover the same result so first entry is a real + // pointer, not an uninitialized InputArg. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("obj", Type::Ref, 8); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 99), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![InputArg::new_ref(0), InputArg::new_ref(1)]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_ref(1)]; + let live_from_entry = vec![(OpRef::input_arg_ref(99), 0)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[], + 1, + 0, + &mut constants, + &entry_mints, + &live_from_entry, + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert!( + ops.iter().any(|op| op.opcode == OpCode::GetfieldGcR), + "must GETFIELD the entry slot the live leftover came from" + ); + let label_args: Vec = ops + .iter() + .find(|op| op.opcode == OpCode::Label) + .expect("label") + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect(); + assert_eq!(label_args[0], OpRef::input_arg_ref(0)); + assert!( + !label_args[1].is_input_arg(), + "entry obj must be a GETFIELD result, got {label_args:?}" + ); + assert_eq!( + label_args[2], label_args[1], + "live leftover must share the GETFIELD pointer, got {label_args:?}" + ); + } + + #[test] + fn test_patch_new_loop_mint_identity_wins_over_positional_getfield() { + // leftover-empty. Two Ref fields. InputArg(2) is mint field 0 + // (the list), not field 1. Positional remap would GETFIELD field 1 + // (a frame). Mint identity must win. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("pattern", Type::Ref, 8); + vinfo.add_field("scratch", Type::Ref, 16); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(24)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 2), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_ref(2), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_ref(50), OpRef::input_arg_ref(51)]; + let live_from_entry = vec![(OpRef::input_arg_ref(2), 0)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[], + 1, + 0, + &mut constants, + &entry_mints, + &live_from_entry, + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + let getfields: Vec = ops + .iter() + .filter(|op| op.opcode == OpCode::GetfieldGcR) + .map(|op| op.pos().get()) + .collect(); + assert_eq!( + getfields.len(), + 2, + "two static Ref fields, got {getfields:?}" + ); + let label_args: Vec = ops + .iter() + .find(|op| op.opcode == OpCode::Label) + .expect("label") + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect(); + assert_eq!( + label_args[2], getfields[0], + "InputArg(2) is mint field 0, not positional field 1, got {label_args:?} getfields={getfields:?}" + ); + assert_ne!( + label_args[2], getfields[1], + "must not leave InputArg(2) on the positional scratch field" + ); + } + + #[test] + fn test_patch_new_loop_reloads_listiter_from_live_tos() { + // leftover ListIter must GETARRAYITEM(valuestackdepth - 1), not a + // baked locals slot (field 26 = `_compile.p` under expanded-tail). + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 99), + ], + ); + let mut seq = Op::new( + OpCode::GetfieldGcR, + &[rooted_inputarg_operand(Type::Ref, 99)], + ); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let mut ops: Vec = vec![label, seq].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + let live_from_entry = vec![(OpRef::input_arg_ref(99), LISTITER_TOS_RELOAD)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &live_from_entry, + Some(vec![0]), + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0), InputArg::new_ref(1)]); + let peel = &ops[leftover_peel_index(&ops)]; + let expected_peel = if cfg!(target_arch = "wasm32") { + leftover_peel_tos_i64 as usize as i64 + } else { + leftover_peel_tos as usize as i64 + }; + assert_eq!( + peel.arg(0).to_opref(), + OpRef::const_int(expected_peel), + "CallR target must be leftover_peel_tos on native, leftover_peel_tos_i64 on wasm" + ); + let seq_recv = ops + .iter() + .find(|op| { + op.opcode == OpCode::GetfieldGcR + && op.getdescr().is_some_and(|d| { + d.as_field_descr() + .is_some_and(crate::history::is_list_iter_seq_field) + }) + }) + .expect("ListIter.seq getfield") + .arg(0) + .to_opref(); + assert_eq!( + seq_recv, + peel.pos().get(), + "ListIter leftover must be leftover_peel_tos, got {seq_recv:?}" + ); + } + + #[test] + fn test_patch_new_loop_reloads_listiter_from_inlined_callee_tos() { + // Portal TOS is the inlined `_compile` frame. leftover-empty + // must GETARRAYITEM that frame's TOS, not use the frame as + // the iterator (`'frame' object is not an iterator`). + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 99), + ], + ); + let mut seq = Op::new( + OpCode::GetfieldGcR, + &[rooted_inputarg_operand(Type::Ref, 99)], + ); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let mut ops: Vec = vec![label, seq].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + let live_from_entry = vec![(OpRef::input_arg_ref(99), LISTITER_TOS_RELOAD)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &live_from_entry, + Some(vec![0, 1]), + ); + + let seq_recv = ops + .iter() + .find(|op| { + op.opcode == OpCode::GetfieldGcR + && op.getdescr().is_some_and(|d| { + d.as_field_descr() + .is_some_and(crate::history::is_list_iter_seq_field) + }) + }) + .expect("ListIter.seq getfield") + .arg(0) + .to_opref(); + let peel = &ops[leftover_peel_index(&ops)]; + assert_eq!( + peel.arg(1).to_opref(), + OpRef::input_arg_ref(0), + "without inline_vable, leftover_peel_tos walks the portal red" + ); + assert_eq!( + seq_recv, + peel.pos().get(), + "ListIter leftover must be leftover_peel_tos, got {seq_recv:?}" + ); + } + + #[test] + fn test_patch_new_loop_peels_inlined_compile_box_not_portal() { + // leftover-empty must leftover_peel_tos the inlined `_compile` + // box (a body New/Getfield), not the portal red. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + + let compile_frame = OpRef::ref_op(100); + let mut new_frame = Op::new( + OpCode::GetfieldGcR, + &[rooted_inputarg_operand(Type::Ref, 0)], + ); + new_frame.pos().set(compile_frame); + new_frame.setdescr(majit_ir::descr::make_field_descr( + 0, + 8, + Type::Ref, + majit_ir::ArrayFlag::Pointer, + )); + let mut seq = Op::new( + OpCode::GetfieldGcR, + &[rooted_inputarg_operand(Type::Ref, 5)], + ); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let mut ops: Vec = vec![ + OpRc::new(Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 5), + ], + )), + OpRc::new(new_frame), + OpRc::new(seq), + ]; + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + ]; + patch_new_loop_to_load_virtualizable_fields_with_vable( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &[(OpRef::input_arg_ref(5), LISTITER_TOS_RELOAD)], + Some(vec![0]), + std::ptr::null(), + Some(compile_frame), + ); + let peel = &ops[leftover_peel_index(&ops)]; + let peel_vable = peel.arg(1).to_opref(); + assert_ne!( + peel_vable, + OpRef::input_arg_ref(0), + "must not peel the portal red" + ); + let frame_load = ops + .iter() + .find(|op| op.pos().get() == peel_vable) + .expect("peel vable must be a produced op"); + assert_eq!(frame_load.opcode, OpCode::GetfieldGcR); + assert!( + !frame_load.getdescr().is_some_and(|d| { + d.as_field_descr() + .is_some_and(crate::history::is_list_iter_seq_field) + }), + "peel vable must be the inlined _compile box, not ListIter.seq" + ); + } + + #[test] + fn test_patch_new_loop_reloads_foriter_residual_from_live_tos() { + // Specialize declined: leftover is the ForIterNext residual arg, + // not a Getfield(seq). Still reload live TOS. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + + let mut effect = majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CanRaise, + majit_ir::OopSpecIndex::None, + ); + effect.runtime_helper = majit_ir::RuntimeHelperKind::ForIterNext; + let descr = majit_ir::descr::make_call_descr(vec![Type::Ref], Type::Ref, effect); + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 5), + ], + ); + let mut call = Op::new(OpCode::CallR, &[rooted_inputarg_operand(Type::Ref, 5)]); + call.setdescr(descr); + let mut ops: Vec = vec![label, call].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &[], + Some(vec![0]), + ); + + let peel = &ops[leftover_peel_index(&ops)]; + let call_arg = ops + .iter() + .find(|op| op.opcode == OpCode::CallR && op.num_args() == 1) + .expect("ForIterNext residual") + .arg(0) + .to_opref(); + assert_eq!( + call_arg, + peel.pos().get(), + "ForIterNext leftover off the TOS field must bind to leftover_peel_tos, got {call_arg:?}" + ); + } + + #[test] + fn test_patch_new_loop_reloads_foriter_at_peeled_portal_tos() { + // Portal TOS index equals the leftover field (`InputArg(6)` = + // prefix+n_static+0). After a peel that slot is the callee + // frame; leftover must follow the inner TOS, not stay on + // GETARRAYITEM(0). + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + + let mut effect = majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CanRaise, + majit_ir::OopSpecIndex::None, + ); + effect.runtime_helper = majit_ir::RuntimeHelperKind::ForIterNext; + let descr = majit_ir::descr::make_call_descr(vec![Type::Ref], Type::Ref, effect); + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 6), + ], + ); + let mut call = Op::new(OpCode::CallR, &[rooted_inputarg_operand(Type::Ref, 6)]); + call.setdescr(descr); + let mut ops: Vec = vec![label, call].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &[], + Some(vec![0, 0]), + ); + + let peel = &ops[leftover_peel_index(&ops)]; + let call_arg = ops + .iter() + .find(|op| op.opcode == OpCode::CallR && op.num_args() == 1) + .expect("ForIterNext residual") + .arg(0) + .to_opref(); + assert_eq!( + call_arg, + peel.pos().get(), + "ForIterNext leftover at the peeled portal TOS must be leftover_peel_tos, got {call_arg:?}" + ); + } + + #[test] + fn test_patch_new_loop_drops_tos_past_the_mint_array() { + // Heap vsd can sit past the minted array (pip: vsd=30, mint=28). + // Do not GETARRAYITEM that slot. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + + let mut seq = Op::new( + OpCode::GetfieldGcR, + &[rooted_inputarg_operand(Type::Ref, 35)], + ); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 35), + ], + ); + let mut ops: Vec = vec![label, seq].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &[], + Some(vec![29]), + ); + + let oob = ops.iter().any(|op| { + matches!( + op.opcode, + OpCode::GetarrayitemGcR | OpCode::GetarrayitemRawR + ) && op.arg(1).to_opref() == OpRef::const_int(29) + }); + assert!(!oob, "must not GETARRAYITEM past the mint array"); + } + + static PEEL_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + #[test] + fn test_leftover_peel_tos_walks_inlined_callee_frame() { + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + register_leftover_scan_frame(std::ptr::null()); + // Portal TOS is an inlined `_compile` frame; that frame's TOS + // is the iterator. leftover_peel_tos must return the iterator, + // not the callee frame. + #[repr(C)] + struct Container { + items: *mut usize, + len: usize, + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Container, + } + const FRAME_TY: usize = 0xF1; + const FRAME_CLASS: usize = 0xF2; + const ITER_TY: usize = 0xA1; + let prev = LISTITER_TYPE_WORD.swap(ITER_TY, std::sync::atomic::Ordering::Relaxed); + struct Restore(usize); + impl Drop for Restore { + fn drop(&mut self) { + LISTITER_TYPE_WORD.store(self.0, std::sync::atomic::Ordering::Relaxed); + } + } + let _restore = Restore(prev); + let mut iterator = [ITER_TY, 0]; + let mut callee_slots = [iterator.as_mut_ptr() as usize]; + let mut callee_arr = Container { + items: callee_slots.as_mut_ptr(), + len: 1, + }; + let mut callee = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut callee_arr, + }; + let mut portal_slots = [&mut callee as *mut Frame as usize]; + let mut portal_arr = Container { + items: portal_slots.as_mut_ptr(), + len: 1, + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut portal_arr, + }; + let got = unsafe { + leftover_peel_tos(&mut portal as *mut Frame as *const u8, 24, 40, 0, 8, 0, 1) + }; + assert_eq!( + got as usize, + iterator.as_mut_ptr() as usize, + "must peel the callee frame and return the iterator" + ); + } + + #[test] + fn test_leftover_peel_tos_direct_pointer_array() { + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + register_leftover_scan_frame(std::ptr::null()); + #[repr(C)] + struct Block { + len: usize, + items: [usize; 1], + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Block, + } + const FRAME_TY: usize = 0xF1; + const FRAME_CLASS: usize = 0xF2; + const ITER_TY: usize = 0xA1; + let prev = LISTITER_TYPE_WORD.swap(ITER_TY, std::sync::atomic::Ordering::Relaxed); + struct Restore(usize); + impl Drop for Restore { + fn drop(&mut self) { + LISTITER_TYPE_WORD.store(self.0, std::sync::atomic::Ordering::Relaxed); + } + } + let _restore = Restore(prev); + let mut iterator = [ITER_TY, 0]; + let mut callee_block = Block { + len: 1, + items: [iterator.as_mut_ptr() as usize], + }; + let mut callee = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut callee_block, + }; + let mut portal_block = Block { + len: 1, + items: [&mut callee as *mut Frame as usize], + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut portal_block, + }; + let items_off = std::mem::offset_of!(Block, items) as i64; + let got = unsafe { + leftover_peel_tos( + &mut portal as *mut Frame as *const u8, + 24, + 40, + 0, + 0, + items_off, + 0, + ) + }; + assert_eq!( + got as usize, + iterator.as_mut_ptr() as usize, + "DirectPointer peel must return the iterator, not the callee frame" + ); + } + + #[test] + fn test_leftover_peel_tos_stops_at_non_frame_tos() { + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + register_leftover_scan_frame(std::ptr::null()); + #[repr(C)] + struct Block { + len: usize, + items: [usize; 1], + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Block, + } + const FRAME_TY: usize = 0xF1; + const FRAME_CLASS: usize = 0xF2; + const STR_TY: usize = 0x53; + let mut string = [STR_TY, 0]; + let mut portal_block = Block { + len: 1, + items: [string.as_mut_ptr() as usize], + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut portal_block, + }; + let items_off = std::mem::offset_of!(Block, items) as i64; + let got = unsafe { + leftover_peel_tos( + &mut portal as *mut Frame as *const u8, + 24, + 40, + 0, + 0, + items_off, + 0, + ) + }; + assert!( + got.is_null(), + "non-listiter TOS must not be handed to FOR_ITER" + ); + } + + #[test] + fn test_leftover_peel_tos_does_not_scan_locals_for_listiter() { + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + register_leftover_scan_frame(std::ptr::null()); + // 31/29 leftover-empty: portal `_compile` TOS is ZipInfo + // (`for op, av in pattern` after unpack) while the live + // listiter sits at a lower locals_cells_stack_w slot. + #[repr(C)] + struct Block { + len: usize, + items: [usize; 3], + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Block, + } + const FRAME_TY: usize = 0xF1; + const FRAME_CLASS: usize = 0xF2; + const ZIPINFO_TY: usize = 0x5A; + const LISTITER_TY: usize = 0x1A11; + let prev = LISTITER_TYPE_WORD.swap(LISTITER_TY, std::sync::atomic::Ordering::Relaxed); + struct Restore(usize); + impl Drop for Restore { + fn drop(&mut self) { + LISTITER_TYPE_WORD.store(self.0, std::sync::atomic::Ordering::Relaxed); + } + } + let _restore = Restore(prev); + let mut zipinfo = [ZIPINFO_TY, 0]; + let mut listiter = [LISTITER_TY, 0]; + let mut other = [0x11usize, 0]; + let mut portal_block = Block { + len: 3, + items: [ + other.as_mut_ptr() as usize, + listiter.as_mut_ptr() as usize, + zipinfo.as_mut_ptr() as usize, + ], + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 3, + _pad2: [0], + locals: &mut portal_block, + }; + let items_off = std::mem::offset_of!(Block, items) as i64; + let got = unsafe { + leftover_peel_tos( + &mut portal as *mut Frame as *const u8, + 24, + 40, + 0, + 0, + items_off, + 0, + ) + }; + assert!( + got.is_null(), + "non-iterator TOS must not invent a listiter from another slot" + ); + } + + #[test] + fn test_leftover_peel_tos_uses_scan_frame_when_portal_tos_is_not_listiter() { + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + // 31/29 leftover-empty: portal red TOS is ZipInfo; the live + // listiter sits on the inlined `_compile` published as the + // leftover scan frame (EC top). + #[repr(C)] + struct Block { + len: usize, + items: [usize; 2], + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Block, + } + const FRAME_TY: usize = 0xF1; + const FRAME_CLASS: usize = 0xF2; + const ZIPINFO_TY: usize = 0x5A; + const LISTITER_TY: usize = 0x1A12; + let prev_ty = LISTITER_TYPE_WORD.swap(LISTITER_TY, std::sync::atomic::Ordering::Relaxed); + let prev_scan = leftover_scan_frame(); + register_leftover_scan_frame(std::ptr::null()); + struct Restore { + ty: usize, + scan: *mut u8, + } + impl Drop for Restore { + fn drop(&mut self) { + LISTITER_TYPE_WORD.store(self.ty, std::sync::atomic::Ordering::Relaxed); + register_leftover_scan_frame(self.scan); + } + } + let _restore = Restore { + ty: prev_ty, + scan: prev_scan, + }; + let mut zipinfo = [ZIPINFO_TY, 0]; + let mut listiter = [LISTITER_TY, 0]; + let mut portal_block = Block { + len: 1, + items: [zipinfo.as_mut_ptr() as usize, 0], + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut portal_block, + }; + let mut compile_block = Block { + len: 2, + items: [0, listiter.as_mut_ptr() as usize], + }; + let mut compile = Frame { + ty: FRAME_TY, + class: FRAME_CLASS, + _pad: [0], + vsd: 2, + _pad2: [0], + locals: &mut compile_block, + }; + register_leftover_scan_frame(&mut compile as *mut Frame as *const u8); + let items_off = std::mem::offset_of!(Block, items) as i64; + let got = unsafe { + leftover_peel_tos( + &mut portal as *mut Frame as *const u8, + 24, + 40, + 0, + 0, + items_off, + 0, + ) + }; + assert_eq!( + got as usize, + listiter.as_mut_ptr() as usize, + "portal ZipInfo TOS must yield the scan-frame listiter" + ); + } + + #[test] + fn test_patch_new_loop_rejects_non_iterator_portal_tos() { + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + register_leftover_scan_frame(std::ptr::null()); + #[repr(C)] + struct Block { + len: usize, + items: [usize; 1], + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Block, + } + const FRAME_TY: usize = 0xF1; + const ZIPINFO_TY: usize = 0x5A; + const LISTITER_TY: usize = 0x1A13; + let prev = LISTITER_TYPE_WORD.swap(LISTITER_TY, std::sync::atomic::Ordering::Relaxed); + struct Restore(usize); + impl Drop for Restore { + fn drop(&mut self) { + LISTITER_TYPE_WORD.store(self.0, std::sync::atomic::Ordering::Relaxed); + let _ = take_leftover_empty_reject(); + } + } + let _restore = Restore(prev); + let _ = take_leftover_empty_reject(); + let mut zipinfo = [ZIPINFO_TY, 0]; + let mut portal_block = Block { + len: 1, + items: [zipinfo.as_mut_ptr() as usize], + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_TY, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut portal_block, + }; + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + std::mem::offset_of!(Block, items), + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 99), + ], + ); + let mut seq = Op::new( + OpCode::GetfieldGcR, + &[rooted_inputarg_operand(Type::Ref, 99)], + ); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let mut ops: Vec = vec![label, seq].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + let live_from_entry = vec![(OpRef::input_arg_ref(99), LISTITER_TOS_RELOAD)]; + patch_new_loop_to_load_virtualizable_fields_with_vable( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &live_from_entry, + Some(vec![0]), + &mut portal as *mut Frame as *const u8, + None, + ); + assert!( + take_leftover_empty_reject(), + "ZipInfo portal TOS must reject leftover-empty peel" + ); + assert!( + !ops.iter() + .any(|op| op.opcode == OpCode::CallR && op.num_args() == 8), + "rejected leftover-empty must not emit leftover_peel_tos" + ); + } + + #[test] + fn test_patch_new_loop_foriter_only_non_listiter_tos_does_not_reject() { + // ForIterNext leftover below TOS is not a ListIter extra. A + // ZipInfo / rangeiter portal TOS must leave it positional, not + // abort the compile (that +1 loops_aborted SNAPDIFF'd unrelated + // range-for traces). + let _guard = PEEL_TEST_LOCK.lock().unwrap(); + register_leftover_scan_frame(std::ptr::null()); + #[repr(C)] + struct Block { + len: usize, + items: [usize; 1], + } + #[repr(C)] + struct Frame { + ty: usize, + class: usize, + _pad: [usize; 1], + vsd: usize, + _pad2: [usize; 1], + locals: *mut Block, + } + const FRAME_TY: usize = 0xF1; + const ZIPINFO_TY: usize = 0x5A; + const LISTITER_TY: usize = 0x1A13; + let prev = LISTITER_TYPE_WORD.swap(LISTITER_TY, std::sync::atomic::Ordering::Relaxed); + struct Restore(usize); + impl Drop for Restore { + fn drop(&mut self) { + LISTITER_TYPE_WORD.store(self.0, std::sync::atomic::Ordering::Relaxed); + let _ = take_leftover_empty_reject(); + } + } + let _restore = Restore(prev); + let _ = take_leftover_empty_reject(); + let mut zipinfo = [ZIPINFO_TY, 0]; + let mut portal_block = Block { + len: 1, + items: [zipinfo.as_mut_ptr() as usize], + }; + let mut portal = Frame { + ty: FRAME_TY, + class: FRAME_TY, + _pad: [0], + vsd: 1, + _pad2: [0], + locals: &mut portal_block, + }; + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_field("pycode", Type::Ref, 16); + vinfo.add_field("valuestackdepth", Type::Int, 24); + vinfo.add_field("debugdata", Type::Ref, 32); + vinfo.add_array_field( + "locals_cells_stack_w", + Type::Ref, + 40, + 0, + std::mem::offset_of!(Block, items), + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(48)); + let mut effect = majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CanRaise, + majit_ir::OopSpecIndex::None, + ); + effect.runtime_helper = majit_ir::RuntimeHelperKind::ForIterNext; + let descr = majit_ir::descr::make_call_descr(vec![Type::Ref], Type::Ref, effect); + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 5), + ], + ); + let mut call = Op::new(OpCode::CallR, &[rooted_inputarg_operand(Type::Ref, 5)]); + call.setdescr(descr); + let mut ops: Vec = vec![label, call].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_int(2), + InputArg::new_ref(3), + InputArg::new_int(4), + InputArg::new_ref(5), + InputArg::new_ref(6), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), + OpRef::input_arg_ref(5), + OpRef::input_arg_ref(6), + ]; + patch_new_loop_to_load_virtualizable_fields_with_vable( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 2, + 0, + &mut constants, + &entry_mints, + &[], + Some(vec![0]), + &mut portal as *mut Frame as *const u8, + None, + ); + assert!( + !take_leftover_empty_reject(), + "ForIterNext-only leftover must not abort leftover-empty compile" + ); + assert!( + !ops.iter() + .any(|op| op.opcode == OpCode::CallR && op.num_args() == 8), + "non-listiter TOS must leave ForIterNext leftover positional" + ); + } + + fn test_patch_new_loop_rejects_leftover_empty_type_mismatch() { + // leftover-empty, length matches the mint (one Ref field) but the + // present slot is an Int. That is a live virtualstate box, not + // `obj`. Do not GETFIELD it as a Ref. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("obj", Type::Ref, 8); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Int, 1), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![InputArg::new_ref(0), InputArg::new_int(1)]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![OpRef::input_arg_ref(1)]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[], + 1, + 0, + &mut constants, + &entry_mints, + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert!( + ops.iter().all(|op| op.opcode != OpCode::GetfieldGcR), + "a type-mismatched leftover-empty tail must not GETFIELD" + ); + } + + #[test] + fn test_patch_new_loop_uses_entry_mints_not_stale_baked_length() { + // leftover-empty. Baked array length is 1 (expanded = prefix + + // last_instr + 1 item = 3) but `entry_field_oprefs` minted + // last_instr + 3 items and inputargs still holds that tail. + // Trusting the bake would long-tail from slot 3 and GETARRAYITEM + // once (binary_slice frame-as-iter). Walk the mint list. + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_field("last_instr", Type::Int, 8); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 16, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(24)); + + let label = Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Int, 1), + rooted_inputarg_operand(Type::Ref, 2), + rooted_inputarg_operand(Type::Ref, 3), + rooted_inputarg_operand(Type::Ref, 4), + ], + ); + let mut ops: Vec = vec![label].into_iter().map(OpRc::new).collect(); + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_int(1), + InputArg::new_ref(2), + InputArg::new_ref(3), + InputArg::new_ref(4), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + let entry_mints = vec![ + OpRef::input_arg_int(1), + OpRef::input_arg_ref(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_ref(4), + ]; + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 1, + 0, + &mut constants, + &entry_mints, + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert_eq!( + ops.iter() + .filter(|op| op.opcode == OpCode::GetarrayitemRawR) + .count(), + 3, + "entry minted 3 array items; baked length 1 must not win" + ); + assert!( + ops.iter().any(|op| op.opcode == OpCode::GetfieldGcI), + "must GETFIELD last_instr" + ); + } + + #[test] + fn test_patch_new_loop_reads_embedded_array_items_from_backing_storage() { + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 8, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let mut ops = vec![Op::new( + OpCode::Label, + &[ + rooted_inputarg_operand(Type::Ref, 0), + rooted_inputarg_operand(Type::Ref, 1), + rooted_inputarg_operand(Type::Ref, 2), + ], + )]; + let mut inputargs = vec![ + InputArg::new_ref(0), + InputArg::new_ref(1), + InputArg::new_ref(2), + ]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + + let mut ops: Vec = ops.into_iter().map(OpRc::new).collect(); + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[2], + 1, + 0, + &mut constants, + &[], + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + assert_eq!(ops.len(), 5); + assert_eq!(ops[0].opcode, OpCode::GetfieldGcR); + assert_eq!(ops[1].opcode, OpCode::GetfieldGcI); + assert_eq!( + ops[1] + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![ops[0].pos().get()] + ); + assert_eq!(ops[2].opcode, OpCode::GetarrayitemRawR); + assert_eq!(ops[2].arg(0).to_opref(), ops[1].pos().get()); + assert_eq!(ops[3].opcode, OpCode::GetarrayitemRawR); + assert_eq!(ops[3].arg(0).to_opref(), ops[1].pos().get()); + assert_eq!(ops[4].opcode, OpCode::Label); + assert_eq!( + ops[4] + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![ + OpRef::input_arg_ref(0), + ops[2].pos().get(), + ops[3].pos().get() + ] + ); + } + + /// Residual body-LABEL `RefOp`s that still forward to an expanded + /// inputarg are the same Box as `loop.inputargs[i]`. `emit_op` must + /// rewrite them to the GETARRAYITEM, not leave a producerless hole. + #[test] + fn test_patch_new_loop_rewrites_residual_label_refop_forwarded_to_inputarg() { + use crate::history::test_support::{rooted_inputarg_operand, rooted_resop_operand}; + + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_embedded_array_field( + "locals_cells_stack_w", + Type::Ref, + 8, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Ref), + ); + vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); + + let slot = rooted_inputarg_operand(Type::Ref, 1); + let reminted = rooted_resop_operand(Type::Ref, 85); + reminted.set_forwarded_inputarg( + &slot + .bound_inputarg() + .expect("rooted inputarg must carry its InputArgRc"), + ); + + let mut ops: Vec = vec![OpRc::new(Op::new( + OpCode::Label, + &[rooted_inputarg_operand(Type::Ref, 0), reminted], + ))]; + let mut inputargs = vec![InputArg::new_ref(0), InputArg::new_ref(1)]; + let mut constants: majit_ir::ConstMap = majit_ir::ConstMap::default(); + + patch_new_loop_to_load_virtualizable_fields( + &mut ops, + &mut inputargs, + &vinfo, + &[1], + 1, + 0, + &mut constants, + &[], + &[], + None, + ); + + assert_eq!(inputargs, vec![InputArg::new_ref(0)]); + let label = ops.iter().find(|op| op.opcode == OpCode::Label).unwrap(); + let getitem = ops + .iter() + .find(|op| op.opcode == OpCode::GetarrayitemRawR) + .unwrap(); + assert_eq!( + label + .getarglist() + .iter() + .map(|a| a.to_opref()) + .collect::>(), + vec![OpRef::input_arg_ref(0), getitem.pos().get()] + ); + } + + /// The entry prefix is the caller's number, not `num_red_args`. + /// + /// A flat entry contract puts the virtualizable at some interior slot with + /// other live entry values around it — here one int scalar at slot 0, the + /// virtualizable at slot 1, and the array elements following from slot 2. + /// `compile.py:431-432` must truncate at 2 and start reconstructing there; + /// a helper that re-derived the point from a red count (this driver + /// declares one red, the whole state) would cut at 1 and reinterpret the + /// virtualizable itself as the first array element. + #[test] + fn test_patch_new_loop_truncates_at_the_callers_entry_prefix_not_a_red_count() { + let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); + vinfo.add_embedded_array_field( + "regs", + Type::Int, + 8, + 0, + 8, + 0, + majit_ir::descr::make_array_descr(0, 8, Type::Int), ); vinfo.set_parent_descr(majit_ir::descr::make_size_descr(16)); @@ -3217,6 +6295,9 @@ mod tests { 2, 1, &mut constants, + &[], + &[], + None, ); // The scalar at slot 0 survives the truncation alongside the @@ -4724,16 +7805,10 @@ impl FailDescr for ResumeGuardCopiedDescr { } /// Per-emission, and deliberately NOT /// chased through `prev`: the classification describes this guard's own - /// condition chain, which a sharer does not inherit from its donor. A - /// copied descr always answers `false`, and that is correct twice over. - /// A poll guard can never be the *sharer*: sharing requires - /// `!op.has_descr() && op.rd_resume_position < 0` (optimizer.rs - /// `emit_guard_operation`, mirrored in optimizeopt/mod.rs), whereas the - /// poll is emitted by `close_loop_args_at` through `generate_guard`, - /// which captures resume data and so always carries a resume position. - /// And when a poll guard is the *donor*, the sharer is some descrless - /// optimizer-created follow-up guard that is not itself a poll, so - /// reading through `prev` would misreport it as one. + /// condition chain, which a sharer does not inherit from its donor. + /// Sharing redirects resume storage, not the classification of this + /// guard's own condition. A poll donor must not classify every sharing + /// guard as a poll; a copied poll is stamped independently at emission. fn is_back_edge_poll(&self) -> bool { self.back_edge_poll .load(std::sync::atomic::Ordering::Relaxed) diff --git a/majit/majit-metainterp/src/history.rs b/majit/majit-metainterp/src/history.rs index cb2c09f7e01..e5eeca8fe56 100644 --- a/majit/majit-metainterp/src/history.rs +++ b/majit/majit-metainterp/src/history.rs @@ -9,6 +9,15 @@ use majit_ir::{DescrRef, InputArg, InputArgRc, Op, OpCode, OpRc, OpRef, Type, Va use parking_lot::Mutex; use std::sync::Arc; +/// Field descr of `W_ListIterObject.seq` / `W_TupleIterObject.seq` / +/// `W_SeqIterObject.seq`. The FOR_ITER list specialize inlines a +/// Getfield of this field; leftover-empty must bind the receiver +/// (the iterator), not the stored list. +pub(crate) fn is_list_iter_seq_field(f: &dyn majit_ir::descr::FieldDescr) -> bool { + let n = f.field_name(); + n.ends_with(".seq") || n.ends_with("::seq") || n == "seq" +} + /// history.py get_const_ptr_for_string(s) /// /// Creates a constant GcRef from byte-string character values. @@ -2727,9 +2736,40 @@ impl TraceCtx { args: &[OpRef], descr: DescrRef, ) -> OpRef { + if opcode.is_call() + && descr.as_call_descr().is_some_and(|cd| { + cd.get_extra_info().runtime_helper == majit_ir::RuntimeHelperKind::GetIter + }) + { + self.note_getiter_iterable(args); + } else if opcode == OpCode::GetfieldGcR + && !args.is_empty() + && descr.as_field_descr().is_some_and(is_list_iter_seq_field) + { + // pip `_compile` never records `RuntimeHelperKind::GetIter`. + // The inlined path is FOR_ITER of an exact list + // (`try_walker_specialize_for_iter_list`): GuardClass(ListIter) + // then Getfield of `W_ListIterObject.seq`. The receiver is the + // iterator leftover leftover-empty must GETFIELD — not the list + // (`SetfieldGc seq` would record the list and alias next() onto it). + self.note_getiter_iterable(&[args[0]]); + } Self::do_record_op_with_descr(&mut self.recorder, opcode, args, descr) } + /// pyjitpl.py `execute_new_with_vtable`: record the allocation and publish + /// both facts the trace-time heap cache learns from it. + pub fn execute_new_with_vtable(&mut self, descr: DescrRef) -> OpRef { + let known_class = descr.as_size_descr().map(|size| size.vtable() as i64); + let result = + Self::do_record_op_with_descr(&mut self.recorder, OpCode::NewWithVtable, &[], descr); + self.heap_cache.new_object(result); + if let Some(class) = known_class { + self.heap_cache.class_now_known(result, class); + } + result + } + /// Record a guard with auto-generated FailDescr. /// /// `num_live` is the number of live integer values (for the FailDescr). diff --git a/majit/majit-metainterp/src/jitdriver.rs b/majit/majit-metainterp/src/jitdriver.rs index c208d916e3d..3610dfe3dde 100644 --- a/majit/majit-metainterp/src/jitdriver.rs +++ b/majit/majit-metainterp/src/jitdriver.rs @@ -6530,12 +6530,21 @@ impl JitDriver { // exception guard unwinds into its handler instead of resuming // the no-exception continuation. let guard_exc = result.exception.exc_value; + // compile.py ResumeGuardForcedDescr.handle_fail reads + // `cpu.get_savedata_ref(deadframe)` before the blackhole. + // Root the copied AllVirtuals object across the bridge + // attempt and resume construction. + let savedata = result.savedata; drop(result); // The deadframe root died with the grab and the reconstruction // below allocates through the blackhole allocator, so hold the // exception where the frontend's root walker can reach it until // `prepare_resume_from_failure` hands it to the blackhole. let _guard_exc_root = crate::blackhole::GuardExcRoot::park(guard_exc); + let savedata_slot = [savedata.map_or(0, majit_ir::GcRef::as_usize) as i64]; + let _savedata_root = unsafe { + crate::resume::DeadFrameRefRoots::enter(&savedata_slot, |_| savedata.is_some()) + }; // must_compile tick for bridge threshold counting. if crate::majit_log_enabled() { @@ -6752,7 +6761,11 @@ impl JitDriver { .map(|a| a.as_ref() as &dyn crate::resume::VirtualizableInfo), None, // ginfo vable_identity_override, - None, // all_virtuals + descr_arc + .is_guard_forced() + .then(|| savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize))) + .flatten() + .and_then(crate::allvirtuals::reveal), allocator, ); let (mut bh, vable_ptr) = bh; @@ -7929,14 +7942,14 @@ impl JitDriver { /// virtuals through the same resume allocator used by ordinary guard /// failure. In particular, a `jit.virtual_ref` frame must not be decoded /// through `NullAllocator`, or its `forced` writeback remains null. - pub fn force_virtualizable_token(&mut self, token: u64) { + pub fn force_virtualizable_token(&mut self, token: u64, identity_override: Option) { let fallback_alloc = crate::resume::NullAllocator; let allocator: &dyn crate::resume::BlackholeAllocator = self .blackhole_allocator .as_deref() .unwrap_or(&fallback_alloc); self.meta - .force_virtualizable_token_with_allocator(token, allocator); + .force_virtualizable_token_with_allocator(token, identity_override, allocator); } /// framework.py `root_walker.walk_roots` parity: visit every Ref-typed @@ -7968,18 +7981,6 @@ impl JitDriver { self.meta.walk_compile_snapshot_refs(visitor); } - /// GC walker for the forced-virtual caches awaiting a `GUARD_NOT_FORCED`. - /// See `MetaInterp::walk_forced_virtuals_refs`. - pub fn walk_forced_virtuals_refs(&mut self, visitor: impl FnMut(&mut majit_ir::GcRef)) { - self.meta.walk_forced_virtuals_refs(visitor); - } - - /// Drop forced-virtual caches whose owner frame died. - /// See `MetaInterp::prune_forced_virtuals`. - pub fn prune_forced_virtuals(&mut self, classify: &mut dyn FnMut(usize) -> Option) { - self.meta.prune_forced_virtuals(classify); - } - pub fn run_compiled_detailed_keyed( &mut self, green_key: u64, @@ -8250,6 +8251,7 @@ impl JitDriver { // guard failure travels with the GuardFailure outcome so the // blackhole resume can seed it (blackhole.py:1794). let guard_exc = result.exception.exc_value; + let savedata = result.savedata; drop(result); // memmgr.py: keep_loop_alive(loop_token) @@ -8330,6 +8332,7 @@ impl JitDriver { raw_values, exit_layout, guard_exc, + savedata, } } diff --git a/majit/majit-metainterp/src/lib.rs b/majit/majit-metainterp/src/lib.rs index 54a082041e4..9d40b86b25a 100644 --- a/majit/majit-metainterp/src/lib.rs +++ b/majit/majit-metainterp/src/lib.rs @@ -107,6 +107,7 @@ pub fn __majit_struct_type_id_path(module_path: &str, type_path: &str, is_gc_man hasher.finish() } +pub mod allvirtuals; pub mod blackhole; pub mod box_trace; pub(crate) mod call_descr; @@ -185,9 +186,10 @@ pub use call_descr::{ nursery_alloc_effect_info, }; pub use compile::{ - make_fail_descr, make_fail_descr_typed, make_finish_fail_descr_typed, + leftover_peel_tos_i64, make_fail_descr, make_fail_descr_typed, make_finish_fail_descr_typed, make_resume_guard_descr_instance_next_foriter, make_resume_guard_descr_range_foriter, - raw_exit_values, + raw_exit_values, register_leftover_scan_frame, register_listiter_pred, + register_listiter_type_word, take_leftover_empty_reject, }; pub use io_buffer::{ emit_commit_io, encode_decimal_i64, io_buffer_commit, io_buffer_discard, io_buffer_write, diff --git a/majit/majit-metainterp/src/pyjitpl.rs b/majit/majit-metainterp/src/pyjitpl.rs index cd53ea8fc35..402061df3d7 100644 --- a/majit/majit-metainterp/src/pyjitpl.rs +++ b/majit/majit-metainterp/src/pyjitpl.rs @@ -391,7 +391,7 @@ impl OptimizationInfoItem for OpRc { let producers = self .getarglist() .into_iter() - .chain(self.getfailargs().into_iter().flatten()); + .chain(self.guard_fail_args().into_iter().flatten().cloned()); for arg in producers { if arg.is_bound() { arg.clear_forwarded(); @@ -1537,7 +1537,9 @@ fn normalize_root_loop_entry_contract( fn densify_root_loop_inputargs( args: &[OpRef], ops: Vec, -) -> (Vec, Vec) { + mint_fields: &[OpRef], + getiter_vable_fields: &[u32], +) -> (Vec, Vec, Vec<(OpRef, u32)>) { let mut replacements: indexmap::IndexMap = indexmap::IndexMap::new(); let inputargs = args @@ -1559,25 +1561,188 @@ fn densify_root_loop_inputargs( }) .collect(); - let remap = |operand: &majit_ir::operand::Operand| { - replacements - .get(&operand.to_opref()) - .map(majit_ir::operand::Operand::from_bound_inputarg) - .unwrap_or_else(|| operand.clone()) + // RPython Box identity: a body-LABEL list-iterator is not the + // entry field box at the same renamed slot. Mapping both onto + // dense InputArg(k) lets leftover-empty GETFIELD reload that + // vable local (`'frame' object is not an iterator`). Keep a + // distinct live InputArg outside the entry vector. + let is_list_or_iter_field = |op: &majit_ir::OpRc| -> bool { + op.opcode == OpCode::GetfieldGcR + && op.getdescr().is_some_and(|d| { + d.as_field_descr().is_some_and(|f| { + let n = f.field_name(); + n.contains("IterObject") + || n.contains("ListIter") + || n.contains("ListObject") + || n.contains("W_List") + || n.contains("W_Tuple") + }) + }) }; - let ops = ops - .into_iter() - .map(|op| { - let args: majit_ir::resoperation::OpArgVec = - op.getarglist().iter().map(&remap).collect(); - let cloned = OpRc::new(op.copy_and_change(op.opcode, Some(&args), None)); - if let Some(failargs) = op.guard_fail_args() { - cloned.setfailargs(failargs.iter().map(&remap).collect()); + let n_renamed = args.len() as u32; + let mint_of = |src: OpRef| -> Option { + let j = mint_fields.iter().position(|&m| m == src)?; + if src.is_input_arg() && src.raw() < n_renamed { + return None; + } + Some(j as u32) + }; + let mut listiter_boxes: rustc_hash::FxHashSet = rustc_hash::FxHashSet::default(); + for op in &ops { + if is_list_or_iter_field(op) { + if let Some(recv) = op.getarglist().first().map(|a| a.to_opref()) { + if replacements.contains_key(&recv) && mint_of(recv).is_some() { + listiter_boxes.insert(recv); + } + } + } + } + let mut live_idents: indexmap::IndexMap = + indexmap::IndexMap::new(); + let mut next_live = args.len() as u32; + let mut seen_label = 0usize; + let mut remapped = Vec::with_capacity(ops.len()); + for op in ops { + if op.opcode == OpCode::Label { + seen_label += 1; + } + let split_live = seen_label >= 2 + || matches!(op.opcode, OpCode::GuardClass | OpCode::GuardNonnullClass) + || is_list_or_iter_field(&op); + let mut remap_one = |operand: &majit_ir::operand::Operand| { + // Only the renamed root boxes. Following `get_box_replacement` + // here remaps a leftover range-iterator (or any body Ref the + // optimizer parked on the vable red) onto the frame: + // `TypeError: 'frame' object is not an iterator`. Leftover + // *field* InputArgs are remapped in + // `patch_new_loop_to_load_virtualizable_fields`, which knows + // the vable mint list. + let src = operand.to_opref(); + if split_live && listiter_boxes.contains(&src) { + if let Some(dense) = replacements.get(&src) { + let live = live_idents + .entry(src) + .or_insert_with(|| { + let minted = InputArgRc::new(InputArg::from_type(dense.tp, next_live)); + next_live += 1; + minted + }) + .clone(); + return majit_ir::operand::Operand::from_bound_inputarg(&live); + } } - cloned + replacements + .get(&src) + .map(majit_ir::operand::Operand::from_bound_inputarg) + .unwrap_or_else(|| operand.clone()) + }; + let new_args = op + .getarglist() + .iter() + .map(&mut remap_one) + .collect::>(); + let cloned = OpRc::new(op.copy_and_change(op.opcode, Some(&new_args), None)); + if let Some(failargs) = op.guard_fail_args() { + cloned.setfailargs(failargs.iter().map(&mut remap_one).collect()); + } + remapped.push(cloned); + } + // Mint identity: renamed[p] == mint[j] means dense InputArg(p) is + // field j, even when p != prefix+j. leftover-empty applies this + // after the positional walk so GET_ITER of that slot reloads the + // mint field (the per-function list/iterator), not the valuestack + // slot at the same dense index. + // + // GET_ITER leftovers whose renamed box is a new virtualstate + // InputArg still bind to the field recorded at trace time. + // pip `_compile` does not emit `RuntimeHelperKind::GetIter`; the + // inlined FOR_ITER path is Getfield of `W_ListIterObject.seq`. + let is_iter_cursor = |op: &majit_ir::OpRc| -> bool { + let is_getiter = op.opcode.is_call() + && op.getdescr().is_some_and(|d| { + d.as_call_descr().is_some_and(|cd| { + cd.get_extra_info().runtime_helper == majit_ir::RuntimeHelperKind::GetIter + }) + }); + if is_getiter { + return true; + } + op.opcode == OpCode::GetfieldGcR + && op.getdescr().is_some_and(|d| { + d.as_field_descr() + .is_some_and(crate::history::is_list_iter_seq_field) + }) + }; + let mut live_from_entry: Vec<(OpRef, u32)> = args + .iter() + .enumerate() + .filter_map(|(p, &src)| { + let j = mint_fields.iter().position(|&m| m == src)?; + let tp = src.ty().unwrap_or(Type::Ref); + Some((OpRef::input_arg_typed(p as u32, tp), j as u32)) }) .collect(); - (inputargs, ops) + let mut getiter_next = 0usize; + for op in &remapped { + if !is_iter_cursor(op) { + continue; + } + for arg in op.getarglist() { + let src = arg.to_opref(); + if !src.is_input_arg() || src.ty() != Some(Type::Ref) { + continue; + } + if live_from_entry.iter().any(|&(r, _)| r == src) { + continue; + } + let j = mint_of(src).or_else(|| { + let field = *getiter_vable_fields.get(getiter_next)?; + getiter_next += 1; + Some(field) + }); + if let Some(j) = j { + live_from_entry.push((src, j)); + } + } + } + // Split leftovers bind to the same mint field when the source is + // the mint box itself. A match against a dense entry InputArg + // (`InputArg(k)` with k < renamed.len()) is the expanded-tail + // numbering alias — that GETFIELD is a value-stack frame. + for (src, live) in &live_idents { + if let Some(j) = mint_fields.iter().position(|&m| m == *src) { + if !(src.is_input_arg() && src.raw() < n_renamed) { + live_from_entry.push((live.opref(), j as u32)); + } + } + } + // FOR_ITER leftover is the iterator at TOS. Expanded-tail mint + // identity maps it onto a local (`_compile.p`). leftover-empty + // reloads `valuestackdepth - 1` from the live frame instead. + let is_listiter_seq = |op: &majit_ir::OpRc| -> bool { + op.opcode == OpCode::GetfieldGcR + && op.getdescr().is_some_and(|d| { + d.as_field_descr() + .is_some_and(crate::history::is_list_iter_seq_field) + }) + }; + for op in &remapped { + if !is_listiter_seq(op) { + continue; + } + for arg in op.getarglist() { + let src = arg.to_opref(); + if !src.is_input_arg() || src.ty() != Some(Type::Ref) { + continue; + } + if let Some(slot) = live_from_entry.iter_mut().find(|(r, _)| *r == src) { + slot.1 = crate::compile::LISTITER_TOS_RELOAD; + } else { + live_from_entry.push((src, crate::compile::LISTITER_TOS_RELOAD)); + } + } + } + (inputargs, remapped, live_from_entry) } pub(crate) struct CompiledEntry { @@ -2139,32 +2304,25 @@ pub struct MetaInterp { /// Used to derive lengths from the actual object when the interpreter /// does not provide them explicitly. pub(crate) vable_ptr: *const u8, - /// `compile.py` — the virtual cache `handle_async_forcing` - /// produced, kept for the `GUARD_NOT_FORCED` failure that must follow. - /// - /// Upstream hides an `AllVirtuals` instance in the deadframe's - /// `jf_savedata` GCREF slot and `ResumeGuardForcedDescr.handle_fail` - /// fishes it back out. pyre cannot put a Rust value in that slot — the GC - /// traces it as a real object reference (`jitframe_trace` in - /// `majit-backend/src/jitframe.rs`) - /// — so the cache is held here, keyed by the virtualizable that was forced. - /// One entry per frame, overwritten on re-force, exactly like the single - /// `jf_savedata` word. - /// - /// The ptr half is a GC root for as long as the entry lives, walked by - /// [`Self::walk_forced_virtuals_refs`]: `force_all_virtuals` - /// (`resume.py:969-981`) materializes every `rd_virtuals` entry, including - /// ones named only by a resume frame's ref registers, and those are written - /// nowhere else at force time — this `Vec` is their only referent until the - /// `GUARD_NOT_FORCED` failure consumes it. Upstream gets that edge from - /// `jf_savedata` being traced; here it comes from the root walker. - /// - /// An entry the guard never consumes is dropped by - /// [`Self::prune_forced_virtuals`] when its owner frame dies, which is what - /// `jf_savedata` gets for free by living on the deadframe. - pub(crate) forced_virtuals: Vec<(u64, Vec, Vec)>, /// Virtualizable array lengths for trace-entry box layout. pub(crate) vable_array_lengths: Vec, + /// Field/array-item InputArgRefs minted at `initialize_virtualizable`. + /// `patch_new_loop` forwards leftover snapshot boxes through the + /// GETFIELD preamble when virtualstate has already dropped them from + /// `inputargs`. + pub(crate) vable_entry_oprefs: Vec, + /// GET_ITER vable field indices copied off `TraceCtx` before + /// `compile_loop` takes the recorder. densify binds leftovers to these + /// when the virtualstate box is a new InputArg. + pub(crate) getiter_vable_fields: Vec, + /// Live boxes densify split off an entry field slot (ListIter / + /// ListObject). leftover-empty GETFIELD reloads that slot and + /// forwards these boxes the GETFIELD result so first entry is + /// not an uninitialized InputArg. + pub(crate) densify_live_from_entry: Vec<(OpRef, u32)>, + /// IR box whose concrete is the inlined `_compile` frame (`orig_vable` + /// when that is not the portal red). leftover-empty GETFIELDs this. + pub(crate) inline_vable_opref: Option, /// warmspot.py:449 jd.result_type — per-driver static result type. pub(crate) result_type: Type, /// PyPy warmspot.py max_unroll_recursion (default 7). @@ -3223,66 +3381,6 @@ impl MetaInterp { } } - /// GC walker for the forced-virtual caches held in - /// `Self::forced_virtuals`, standing in for the trace `jf_savedata` gets - /// as a real GCREF field (`jitframe_trace` in - /// `majit-backend/src/jitframe.rs`). - /// - /// Only the ptr half is walked. The int half is `virtuals_int_cache` — - /// unboxed integer field values — and handing those to the visitor would - /// test integers as heap addresses. `prepare_resume_heap_with_roots` roots - /// the same one half for the same reason. - /// - /// Unmaterialized `0` cache slots pass through unchanged, as they do in - /// `shadow_stack::walk_resume_ref_roots`. - /// - /// The walk is unconditional, so it is a strong edge where `jf_savedata` is - /// an ephemeron one: a major seeds these values from `seed_major_roots`, - /// while [`Self::prune_forced_virtuals`] can only drop a dead owner's entry - /// at the *end* of marking, once VISITED is decided. An entry whose owner - /// died inside the cycle therefore keeps its materialized graph until the - /// next major — one cycle of floating garbage, bounded: the sweep clears - /// VISITED on every survivor and the entry is gone, and the sole writer - /// (`save_forced_virtuals`) is reached only by forcing a live virtualizable, - /// so nothing re-adds it. The prune also strictly precedes the sweep, so a - /// named owner's address can never be recycled underneath an entry. - /// - /// That bound holds only while no cached virtual reaches the owner frame: a - /// back-edge would let this walk mark the owner, `classify_owner` would then - /// answer `Some`, and the entry would never be pruned at all. The vable is - /// returned beside the cache rather than inside it (`force_from_resumedata`) - /// and no such edge exists today. - pub fn walk_forced_virtuals_refs(&mut self, mut visitor: impl FnMut(&mut GcRef)) { - for (_owner, ptrs, _ints) in self.forced_virtuals.iter_mut() { - for slot in ptrs.iter_mut() { - // SAFETY: `GcRef` is a pointer-sized newtype over the same - // representation these slots hold, the same reinterpret - // `walk_resume_ref_roots` performs on `virtuals_ptr_cache` - // (`majit-gc/src/shadow_stack.rs`). Walked unconditionally - // for both collection kinds: the minor forwards a - // nursery-resident virtual in place, the major seeds it as a - // mark root so the sweep does not free it. - let gcref = unsafe { &mut *(slot as *mut i64 as *mut GcRef) }; - visitor(gcref); - } - } - } - - /// Drop the forced-virtual caches whose owner frame died. - /// - /// `classify` answers with the owner's current address, or `None` if it did - /// not survive. A major collection moves nothing, so a surviving owner - /// answers with the key it was asked about. - /// - /// Without this, rooting the ptr half would keep an unconsumed entry's - /// objects alive forever and leave a stale `PyFrame` key that a later frame - /// at the same address could fish. Upstream is immune because `jf_savedata` - /// dies with its deadframe; this reproduces that lifetime. - pub fn prune_forced_virtuals(&mut self, classify: &mut dyn FnMut(usize) -> Option) { - self.forced_virtuals - .retain(|(owner, _, _)| classify(*owner as usize) == Some(*owner as usize)); - } - #[inline] fn prepare_compiled_run_io() { io_buffer::io_buffer_discard(); @@ -3888,8 +3986,11 @@ impl MetaInterp { pending_token: None, stats: JitStatsCounters::default(), vable_ptr: std::ptr::null(), - forced_virtuals: Vec::new(), vable_array_lengths: Vec::new(), + vable_entry_oprefs: Vec::new(), + getiter_vable_fields: Vec::new(), + densify_live_from_entry: Vec::new(), + inline_vable_opref: None, result_type: Type::Ref, max_unroll_recursion: 7, // RPython default from rlib/jit.py force_finish_trace: false, @@ -4927,6 +5028,13 @@ impl MetaInterp { // pyjitpl.py `initialize_virtualizable` closes by asserting the // freshly read boxes still match the object it read them from. ctx.check_synchronized_virtualizable(); + // Keep the trace-entry lengths and minted field boxes for + // `patch_new_loop`. A later live read can see a shorter valuestack + // and drop InputArgRefs the snapshot still names. + self.set_vable_array_lengths(array_lengths); + self.vable_entry_oprefs = vable_oprefs; + self.getiter_vable_fields.clear(); + self.inline_vable_opref = None; } /// warmstate.py: set_param_trace_eagerness — delegates to warmstate. @@ -7044,10 +7152,6 @@ impl MetaInterp { Some(flat) => flat.len, None => driver.num_reds(), }; - if inputargs.len() <= entry_prefix_len { - // Trace was never expanded (no virtualizable fields live at entry). - return; - } // compile.py:508-511 // vable = orig_inpargs[jitdriver_sd.index_of_virtualizable].getref_base() // patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd, vable) @@ -7071,10 +7175,15 @@ impl MetaInterp { // length on the heap object must be fixed inside // `VirtualizableInfo` itself (to match `vinfo.get_array_length`'s // universal contract), not worked around in this helper. - let array_lengths: Vec = (0..vinfo.array_fields.len()) - .map(|i| unsafe { vinfo.get_array_length(orig_vable_ptr, i) }) - .collect(); - compile::patch_new_loop_to_load_virtualizable_fields( + let array_lengths: Vec = if !self.vable_array_lengths.is_empty() { + self.vable_array_lengths.clone() + } else { + (0..vinfo.array_fields.len()) + .map(|i| unsafe { vinfo.get_array_length(orig_vable_ptr, i) }) + .collect() + }; + let live_tos = unsafe { crate::compile::live_tos_for_vable(vinfo, orig_vable_ptr) }; + compile::patch_new_loop_to_load_virtualizable_fields_with_vable( ops, inputargs, vinfo, @@ -7082,6 +7191,11 @@ impl MetaInterp { entry_prefix_len, index_of_vable, constants, + &self.vable_entry_oprefs, + &self.densify_live_from_entry, + live_tos, + orig_vable_ptr, + self.inline_vable_opref, ); // compile.py `patch_new_loop_to_load_virtualizable_fields` // does not change LABEL/JUMP *arity*. `emit_op` still rewrites @@ -7518,6 +7632,7 @@ impl MetaInterp { } self.force_finish_trace = false; let mut ctx = self.tracing.take().unwrap(); + self.getiter_vable_fields = ctx.getiter_vable_fields.clone(); // Cache driver descriptor before ctx is partially consumed below; // mirrors the FINISH-path capture pattern (see `finish_and_compile`). let driver_descriptor = ctx.driver_descriptor().cloned(); @@ -7542,6 +7657,16 @@ impl MetaInterp { // (compile.py:443). let orig_vable_ptr_loop = self.orig_vable_ptr_for_cut(cut_merge_point, &ctx, driver_descriptor.as_ref()); + let portal_idx = driver_descriptor + .as_ref() + .and_then(|d| d.virtualizable_arg_index()) + .unwrap_or(0); + self.inline_vable_opref = ctx.inline_vable_box().or_else(|| { + ctx.opref_with_concrete_ref( + orig_vable_ptr_loop as usize, + OpRef::input_arg_ref(portal_idx as u32), + ) + }); let cross_loop_cut = cut_merge_point.map(|mp| { ( mp.green_boxes.clone(), @@ -8141,8 +8266,29 @@ impl MetaInterp { } }; let (root_inputargs, mut optimized_ops) = match renamed_root_args { - Some(args) => densify_root_loop_inputargs(&args, optimized_ops), - None => (trace.inputargs_cloned(), optimized_ops), + Some(args) => { + let getiter_fields = if !self.getiter_vable_fields.is_empty() { + self.getiter_vable_fields.as_slice() + } else { + self.tracing + .as_ref() + .or(self.compile_tracing.as_ref()) + .map(|ctx| ctx.getiter_vable_fields.as_slice()) + .unwrap_or(&[]) + }; + let (inputargs, ops, live) = densify_root_loop_inputargs( + &args, + optimized_ops, + &self.vable_entry_oprefs, + getiter_fields, + ); + self.densify_live_from_entry = live; + (inputargs, ops) + } + None => { + self.densify_live_from_entry.clear(); + (trace.inputargs_cloned(), optimized_ops) + } }; if retried_without_unroll && !optimized_ops @@ -8475,6 +8621,12 @@ impl MetaInterp { driver_descriptor.as_ref(), orig_vable_ptr_loop, ); + if crate::compile::take_leftover_empty_reject() { + if crate::majit_log_enabled() || std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!("[jit] leftover-empty reject: portal TOS is not a listiter"); + } + return CompileOutcome::Aborted; + } if crate::majit_log_enabled() { eprintln!( "[jit] pre-backend: {} ops, {} inputargs", @@ -10510,6 +10662,9 @@ impl MetaInterp { // Taking it out of `tracing` stops a re-entrant record; dropping it // before compile intern would reopen the nursery-ConstPtr window. self.compile_tracing = self.tracing.take(); + if let Some(ctx) = self.compile_tracing.as_ref() { + self.getiter_vable_fields = ctx.getiter_vable_fields.clone(); + } let compile_tracing_slot = &raw mut self.compile_tracing; struct CompileTracingGuard(*mut Option); impl Drop for CompileTracingGuard { @@ -10530,6 +10685,19 @@ impl MetaInterp { let ctx = self.compile_tracing.as_ref().unwrap(); self.orig_vable_ptr_from_trace_ctx(ctx, driver_descriptor.as_ref()) }; + { + let ctx = self.compile_tracing.as_ref().unwrap(); + let portal_idx = driver_descriptor + .as_ref() + .and_then(|d| d.virtualizable_arg_index()) + .unwrap_or(0); + self.inline_vable_opref = ctx.inline_vable_box().or_else(|| { + ctx.opref_with_concrete_ref( + orig_vable_ptr as usize, + OpRef::input_arg_ref(portal_idx as u32), + ) + }); + } // pyjitpl.py compile_done_with_this_frame parity: // `store_token_in_vable` (SetfieldGc on vable_token + the // accompanying GUARD_NOT_FORCED_2) is recorded by the pyre @@ -15697,10 +15865,9 @@ impl MetaInterp { /// RPython flow: force_now() → cpu.force(token) → handle_async_forcing() /// → force_from_resumedata() → materialize all virtuals → save on deadframe. /// - /// The forced virtual caches (ptr, int) are stored on - /// `Self::forced_virtuals` for the blackhole resumption from the - /// GUARD_NOT_FORCED — RPython's `AllVirtuals` via `cpu.set_savedata_ref()`. - /// They are also returned, which only the unit tests below read. + /// The returned caches are attached to the backend deadframe by + /// `force_virtualizable_token_with_allocator`, matching `AllVirtuals` / + /// `cpu.set_savedata_ref` in `ResumeGuardForcedDescr.handle_async_forcing`. pub fn handle_async_forcing( &mut self, green_key: u64, @@ -15714,6 +15881,7 @@ impl MetaInterp { trace_id, fail_index, fail_values, + None, &crate::resume::NullAllocator, ) } @@ -15729,6 +15897,7 @@ impl MetaInterp { trace_id: u64, fail_index: u32, fail_values: &[i64], + identity_override: Option, allocator: &dyn crate::resume::BlackholeAllocator, ) -> Option<(Vec, Vec)> { if crate::majit_log_enabled() { @@ -15750,14 +15919,8 @@ impl MetaInterp { None => self.get_compiled_exit_layout_in_trace(green_key, norm_tid, fail_index)?, }; - // compile.py:973-985 don't interrupt me! If the stack runs out - // in force_from_resumedata() then we have seen cpu.force() but - // not self.save_data(), leaving in an inconsistent state. - // - // RPython wraps the body in try/finally. CriticalCodeGuard's - // Drop impl re-enables report_error on every exit — including - // panic unwind — matching the RPython contract. - let _cc_guard = crate::CriticalCodeGuard::enter(); + // `ResumeGuardForcedDescr.force_now` owns the critical interval, + // including the subsequent `AllVirtuals` allocation and publication. // compile.py:994: force_from_resumedata(metainterp_sd, self, deadframe, vinfo, ginfo) // compile.py `ResumeGuardDescr` storage — borrow rd_numb / // rd_consts / rd_virtuals / rd_pendingfields off the guard-owned @@ -15800,7 +15963,7 @@ impl MetaInterp { // compile.py:990-991: vinfo = self.jitdriver_sd.virtualizable_info let vinfo = self.virtualizable_info(); let all_liveness = self.staticdata.liveness_info.as_slice(); - let (all_virtuals_ptr, all_virtuals_int, virtualizable_ptr) = + let (all_virtuals_ptr, all_virtuals_int, _virtualizable_ptr) = crate::resume::force_from_resumedata( &self.staticdata.profiler, rd_numb, @@ -15813,9 +15976,9 @@ impl MetaInterp { Some(&self.staticdata.virtualref_info as &dyn crate::resume::VRefInfo), vinfo.map(|v| v.as_ref() as &dyn crate::resume::VirtualizableInfo), None, // ginfo — pyre has no greenfield mechanism + identity_override, allocator, ); - drop(_cc_guard); if crate::majit_log_enabled() { eprintln!( "[jit][handle_async_forcing] forced {} ptr + {} int virtuals", @@ -15823,20 +15986,6 @@ impl MetaInterp { all_virtuals_int.len(), ); } - // compile.py: obj = AllVirtuals(all_virtuals) - // metainterp_sd.cpu.set_savedata_ref(deadframe, obj.hide()) - // - // The store lives here, inside `handle_async_forcing`, exactly as - // upstream — every force entry point reaching this function is covered - // by it, including `force_virtual_if_necessary`'s (virtualref.py) - // which never sees the returned caches. - if virtualizable_ptr != 0 { - self.save_forced_virtuals( - virtualizable_ptr as u64, - all_virtuals_ptr.clone(), - all_virtuals_int.clone(), - ); - } Some((all_virtuals_ptr, all_virtuals_int)) } @@ -15854,9 +16003,13 @@ impl MetaInterp { pub fn force_virtualizable_token_with_allocator( &mut self, token: u64, + identity_override: Option, allocator: &dyn crate::resume::BlackholeAllocator, ) { - let deadframe = self + // `ResumeGuardForcedDescr.force_now`: the critical interval includes + // publishing savedata, not only materializing the cache. + let _critical = crate::CriticalCodeGuard::enter(); + let mut deadframe = self .backend .force(GcRef(token as usize)) .expect("active virtualizable must have a backend deadframe"); @@ -15881,66 +16034,27 @@ impl MetaInterp { }) .collect::>(); // compile.py: faildescr.handle_async_forcing(deadframe) - self.handle_async_forcing_with_allocator( - Some(descr), - green_key, - trace_id, - fail_index, - &fail_values, - allocator, - ); - } - - /// `compile.py cpu.set_savedata_ref(deadframe, obj.hide())`. - /// - /// `owner` is the forced virtualizable, as the guard's own resume data - /// named it (`resume.py:1404`). Upstream can key on the deadframe because - /// `handle_fail` receives it; pyre's guard-failure path surfaces the frame - /// rather than the jitframe, and the two ends agree on the frame: the force - /// runs against a named virtualizable and the GUARD_NOT_FORCED that follows - /// deopts that same frame's loop. - fn save_forced_virtuals(&mut self, owner: u64, ptrs: Vec, ints: Vec) { - // The key is a bare address, so the mechanism holds only while that - // address is stable. It is: the virtualizable is an interpreter-created - // frame (`FrameBox::new` → `try_gc_alloc_stable_raw`, "stable across - // minor and major collections (MiniMark mark-sweep does not move - // old-gen objects)"), never one of the frames a trace builds virtually. - // A nursery owner would break both ends — a minor would forward the - // frame out from under this key, and the pruner's classifier keeps every - // non-old-gen owner, so the entry could never be dropped either. - debug_assert!( - !majit_gc::gc_is_nursery_object(owner as usize), - "forced-virtual cache keyed on a nursery-resident virtualizable \ - (0x{owner:x}): the key must be a move-stable address", - ); - match self.forced_virtuals.iter_mut().find(|e| e.0 == owner) { - // A second force of the same frame overwrites, the way a second - // `set_savedata_ref` overwrites the one `jf_savedata` word. - Some(entry) => *entry = (owner, ptrs, ints), - None => self.forced_virtuals.push((owner, ptrs, ints)), - } - } - - /// `compile.py` — `handle_fail` of a `GUARD_NOT_FORCED` fishes - /// the cache `handle_async_forcing` left on the deadframe and hands it - /// to `resume_in_blackhole`, which is what makes the blackhole reuse - /// the objects the force already materialized instead of building a - /// second set (`resume.py:1373-1374`, and the `vable_size` skip in - /// `consume_vref_and_vable`). - pub fn take_forced_virtuals(&mut self, owner: u64) -> Option<(Vec, Vec)> { - let index = self.forced_virtuals.iter().position(|e| e.0 == owner); - // Only a GUARD_NOT_FORCED reaches here (`is_guard_forced()` gates the - // callers), so hit/miss is the force→resume handoff itself: the - // counterpart of the `handle_async_forcing` line above. - if crate::majit_log_enabled() { - eprintln!( - "[jit][take_forced_virtuals] owner=0x{:x} {}", - owner, - if index.is_some() { "hit" } else { "miss" }, - ); - } - let (_, ptrs, ints) = self.forced_virtuals.swap_remove(index?); - Some((ptrs, ints)) + let (ptrs, ints) = self + .handle_async_forcing_with_allocator( + Some(descr), + green_key, + trace_id, + fail_index, + &fail_values, + identity_override, + allocator, + ) + .expect("forced guard must have resume data"); + let savedata = crate::allvirtuals::allocate(ptrs, ints); + // compile.py handle_async_forcing writes `deadframe.jf_savedata` + // through a GCREF store. Every managed backend starts that store + // with a write barrier, so root the fresh AllVirtuals object and + // publish the forwarded address. + let savedata_slot = [savedata.as_usize() as i64]; + let _savedata_root = + unsafe { crate::resume::DeadFrameRefRoots::enter(&savedata_slot, |_| true) }; + self.backend + .set_savedata_ref(&mut deadframe, majit_ir::GcRef(savedata_slot[0] as usize)); } pub fn is_force_token_armed(&self, token: u64) -> bool { @@ -19414,6 +19528,9 @@ pub enum DetailedDriverRunOutcome { /// `_prepare_resume_from_failure`) so an exception guard unwinds /// to its handler instead of resuming the no-exception path. guard_exc: i64, + /// `ResumeGuardForcedDescr.handle_fail` consumes the cache saved on + /// this deadframe, never a cache keyed by the virtualizable's address. + savedata: Option, }, Abort { restored: bool, @@ -25145,7 +25262,8 @@ mod tests { .map(majit_ir::operand::Operand::bound_from_opref) .collect(), ); - let (inputargs, ops) = densify_root_loop_inputargs(&renamed, vec![guard]); + let (inputargs, ops, live) = densify_root_loop_inputargs(&renamed, vec![guard], &[], &[]); + assert!(live.is_empty()); assert_eq!( inputargs.iter().map(InputArg::opref).collect::>(), vec![ @@ -25165,6 +25283,257 @@ mod tests { ); } + #[test] + fn test_densify_does_not_follow_forwarded_leftover_onto_the_frame() { + let renamed = vec![OpRef::input_arg_ref(425)]; + let canonical = InputArgRc::new(InputArg::new_ref(425)); + let leftover = InputArgRc::new(InputArg::new_ref(98)); + majit_ir::operand::Operand::from_bound_inputarg(&leftover) + .set_forwarded_inputarg(&canonical); + let guard = OpRc::new(mk_op( + OpCode::GuardClass, + &[renamed[0], OpRef::const_ptr(majit_ir::GcRef(0x1234))], + OpRef::NONE.raw(), + )); + guard.setfailargs( + [majit_ir::operand::Operand::from_bound_inputarg(&leftover)] + .into_iter() + .collect(), + ); + let (inputargs, ops, live) = densify_root_loop_inputargs(&renamed, vec![guard], &[], &[]); + assert!(live.is_empty()); + assert_eq!(inputargs.len(), 1); + assert_eq!(ops[0].arg(0).to_opref(), OpRef::input_arg_ref(0)); + assert_eq!( + ops[0].guard_fail_args().unwrap()[0].to_opref(), + OpRef::input_arg_ref(98), + "a leftover body Ref must not become the vable red" + ); + } + + #[test] + fn test_densify_keeps_body_label_listiter_off_entry_field_slots() { + // renamed entry is frame + field. Body LABEL / GuardClass carry + // the same renamed Ref as the field slot. Densify must not put + // that ListIter on dense InputArg(1): leftover-empty GETFIELD + // would reload the vable local (`'frame' object is not an iterator`). + let renamed = vec![OpRef::input_arg_ref(0), OpRef::input_arg_ref(50)]; + let start = OpRc::new(mk_op( + OpCode::Label, + &[renamed[0], renamed[1]], + OpRef::NONE.raw(), + )); + let guard = OpRc::new(mk_op( + OpCode::GuardClass, + &[renamed[1], OpRef::const_ptr(majit_ir::GcRef(0x1234))], + OpRef::NONE.raw(), + )); + let mut seq = mk_op(OpCode::GetfieldGcR, &[renamed[1]], 99); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let seq = OpRc::new(seq); + let body = OpRc::new(mk_op( + OpCode::Label, + &[renamed[0], renamed[1]], + OpRef::NONE.raw(), + )); + let (inputargs, ops, live) = densify_root_loop_inputargs( + &renamed, + vec![start, guard, seq, body], + &[renamed[1]], + &[], + ); + assert_eq!(inputargs.len(), 2); + assert!( + live.iter().any(|&(r, j)| r.is_input_arg() + && r.raw() >= 2 + && j == crate::compile::LISTITER_TOS_RELOAD), + "ListIter leftover must bind to live TOS reload, got {live:?}" + ); + assert_eq!(ops[0].arg(1).to_opref(), OpRef::input_arg_ref(1)); + let live = ops[1].arg(0).to_opref(); + assert!( + live.is_input_arg() && live.raw() >= 2, + "ListIter must sit outside the entry vector, got {live:?}" + ); + assert_eq!(ops[2].arg(0).to_opref(), live); + assert_eq!(ops[3].arg(1).to_opref(), live); + assert_ne!(live, OpRef::input_arg_ref(1)); + } + + #[test] + fn test_densify_does_not_bind_listiter_to_an_aliased_dense_slot() { + // renamed[1] is a fresh entry InputArg, not the mint field box. + // leftover-empty must not treat dense slot 1 as the iterator field + // (that GETFIELD is a value-stack frame). + let renamed = vec![OpRef::input_arg_ref(0), OpRef::input_arg_ref(50)]; + let start = OpRc::new(mk_op( + OpCode::Label, + &[renamed[0], renamed[1]], + OpRef::NONE.raw(), + )); + let guard = OpRc::new(mk_op( + OpCode::GuardClass, + &[renamed[1], OpRef::const_ptr(majit_ir::GcRef(0x1234))], + OpRef::NONE.raw(), + )); + let mut seq = mk_op(OpCode::GetfieldGcR, &[renamed[1]], 99); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let seq = OpRc::new(seq); + let mint = vec![OpRef::input_arg_ref(80)]; + let (_inputargs, _ops, live) = + densify_root_loop_inputargs(&renamed, vec![start, guard, seq], &mint, &[]); + assert!( + !live.iter().any(|&(r, _)| r.is_input_arg() && r.raw() >= 2), + "ListIter aliased onto a dense slot must not bind to that slot's GETFIELD, got {live:?}" + ); + + let renamed_dense = vec![OpRef::input_arg_ref(0), OpRef::input_arg_ref(1)]; + let start2 = OpRc::new(mk_op( + OpCode::Label, + &[renamed_dense[0], renamed_dense[1]], + OpRef::NONE.raw(), + )); + let guard2 = OpRc::new(mk_op( + OpCode::GuardClass, + &[renamed_dense[1], OpRef::const_ptr(majit_ir::GcRef(0x1234))], + OpRef::NONE.raw(), + )); + let mut seq2 = mk_op(OpCode::GetfieldGcR, &[renamed_dense[1]], 99); + seq2.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let (_ia, _ops, live_dense) = densify_root_loop_inputargs( + &renamed_dense, + vec![start2, guard2, OpRc::new(seq2)], + &[renamed_dense[1]], + &[], + ); + assert!( + !live_dense + .iter() + .any(|&(r, _)| r.is_input_arg() && r.raw() >= 2), + "expanded-tail ListIter leftover must not bind to the aliased slot, got {live_dense:?}" + ); + } + + #[test] + fn test_densify_records_mint_identity_when_the_field_is_not_at_its_dense_index() { + // pattern is mint field 0 sitting at renamed[2], not at prefix+0. + // leftover-empty must GETFIELD field 0 for InputArg(2), not field 2. + let renamed = vec![ + OpRef::input_arg_ref(0), + OpRef::input_arg_ref(10), + OpRef::input_arg_ref(50), + ]; + let start = OpRc::new(mk_op(OpCode::Label, &renamed, OpRef::NONE.raw())); + let (_ia, _ops, live) = + densify_root_loop_inputargs(&renamed, vec![start], &[renamed[2]], &[]); + assert!( + live.iter() + .any(|&(r, j)| r == OpRef::input_arg_ref(2) && j == 0), + "dense InputArg(2) is mint field 0, got {live:?}" + ); + } + + #[test] + fn test_densify_binds_getiter_to_trace_time_vable_field() { + // After virtualstate the GET_ITER arg is a new InputArg, not the + // mint box. leftover-empty must still GETFIELD the field recorded + // when the residual GetIter was traced. + let renamed = vec![ + OpRef::input_arg_ref(0), + OpRef::input_arg_ref(10), + OpRef::input_arg_ref(99), + ]; + let mut effect = majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CanRaise, + majit_ir::OopSpecIndex::None, + ); + effect.runtime_helper = majit_ir::RuntimeHelperKind::GetIter; + let descr = majit_ir::descr::make_call_descr(vec![Type::Ref], Type::Ref, effect); + let mut call = mk_op(OpCode::CallR, &[renamed[2]], 40); + call.setdescr(descr); + let start = OpRc::new(mk_op(OpCode::Label, &renamed, OpRef::NONE.raw())); + let (_ia, _ops, live) = densify_root_loop_inputargs( + &renamed, + vec![start, OpRc::new(call)], + &[OpRef::input_arg_ref(50)], + &[5], + ); + assert!( + live.iter() + .any(|&(r, j)| r == OpRef::input_arg_ref(2) && j == 5), + "GET_ITER leftover InputArg(2) must bind to traced field 5, got {live:?}" + ); + } + + #[test] + fn test_densify_binds_listiter_seq_to_trace_time_vable_field() { + // pip `_compile` leftover is the FOR_ITER ListIter, a new + // virtualstate InputArg. leftover-empty must GETFIELD the + // iterator slot recorded when Getfield(W_ListIterObject.seq) + // was traced, not the dense position (a valuestack frame). + let renamed = vec![ + OpRef::input_arg_ref(0), + OpRef::input_arg_ref(10), + OpRef::input_arg_ref(99), + ]; + let mut seq = mk_op(OpCode::GetfieldGcR, &[renamed[2]], 40); + seq.setdescr(std::sync::Arc::new( + majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + ), + )); + let start = OpRc::new(mk_op(OpCode::Label, &renamed, OpRef::NONE.raw())); + let mint: Vec = (0..8).map(|i| OpRef::input_arg_ref(50 + i)).collect(); + let (_ia, _ops, live) = + densify_root_loop_inputargs(&renamed, vec![start, OpRc::new(seq)], &mint, &[7]); + assert!( + live.iter() + .any(|&(r, j)| r == OpRef::input_arg_ref(2) + && j == crate::compile::LISTITER_TOS_RELOAD), + "ListIter leftover InputArg(2) must bind to live TOS reload, got {live:?}" + ); + } + #[test] fn test_prepare_bridge_trace_for_optimizer_freshens_inputargs_and_snapshots() { let bridge_inputargs = vec![InputArg::new_int(0), InputArg::new_ref(1)]; @@ -27178,9 +27547,9 @@ mod tests { meta.opimpl_hint_force_virtualizable(OpRef::input_arg_ref(0)); let ops = take_recorded_ops(&mut meta); - assert_eq!(ops.len(), 2); + // Unmodified static boxes are skipped; only the token reset remains. + assert_eq!(ops.len(), 1); assert_eq!(ops[0].opcode, OpCode::SetfieldGc); - assert_eq!(ops[1].opcode, OpCode::SetfieldGc); } #[test] @@ -27483,9 +27852,9 @@ mod tests { meta.opimpl_hint_force_virtualizable(OpRef::input_arg_ref(0)); let ops = take_recorded_ops(&mut meta); - assert_eq!(ops.len(), 2); + // Second trace is a fresh init, so the token store is recorded again. + assert_eq!(ops.len(), 1); assert_eq!(ops[0].opcode, OpCode::SetfieldGc); - assert_eq!(ops[1].opcode, OpCode::SetfieldGc); } #[test] @@ -27506,11 +27875,12 @@ mod tests { meta.opimpl_hint_force_virtualizable(OpRef::input_arg_ref(0)); let ops = take_recorded_ops(&mut meta); - assert_eq!(ops.len(), 4); + // First hint writes the token; getfield_vable consumes forced + // state; second hint writes the token again. Static boxes are + // unmodified so they are not stored. + assert_eq!(ops.len(), 2); assert_eq!(ops[0].opcode, OpCode::SetfieldGc); assert_eq!(ops[1].opcode, OpCode::SetfieldGc); - assert_eq!(ops[2].opcode, OpCode::SetfieldGc); - assert_eq!(ops[3].opcode, OpCode::SetfieldGc); } #[test] diff --git a/majit/majit-metainterp/src/resume.rs b/majit/majit-metainterp/src/resume.rs index c3675cb6042..630dfd2ed1e 100644 --- a/majit/majit-metainterp/src/resume.rs +++ b/majit/majit-metainterp/src/resume.rs @@ -4450,7 +4450,13 @@ impl ResumeDataLoopMemo { minimum_virtualizable_size: i64, ) -> Result { let mut frames: SmallVec<[(i32, i32, i32, &[SnapshotBox]); 16]> = SmallVec::new(); - if let Some(sizes) = frame_sizes.filter(|sizes| sizes.len() > 1) { + // An explicitly empty framestack is the terminal force snapshot + // (opencoder.py create_empty_top_snapshot), not an absent layout. + // A one-frame sizes vector is the default single-frame case + // (`number` already emits that layout); keep it on the fallback + // so a size that does not cover every snapshot box cannot drop + // the remainder. Multi-frame and empty stay explicit. + if let Some(sizes) = frame_sizes.filter(|sizes| sizes.len() != 1) { let mut offset = 0; for (i, &size) in sizes.iter().enumerate() { let end = (offset + size).min(snapshot_boxes.len()); @@ -5384,6 +5390,66 @@ mod tests { assert_eq!(val, 1); } + #[test] + fn number_from_parts_preserves_an_empty_framestack_with_vable_state() { + // opencoder.py create_empty_top_snapshot, reached after + // pyjitpl.py finishframe has popped the last MIFrame. + let snapshot = Snapshot { + framestack: Vec::new(), + vable_array: vec![ + SnapshotBox::from(OpRef::ref_op(0)), + SnapshotBox::from(OpRef::const_int(7)), + ], + vref_array: Vec::new(), + }; + let env = SimpleBoxEnv::new(); + let expected = ResumeDataLoopMemo::new() + .number(&snapshot, &env, -1) + .unwrap() + .create_numbering(); + let actual = ResumeDataLoopMemo::new() + .number_from_parts(&[], Some(&[]), &[], &snapshot.vable_array, &[], &env, -1) + .unwrap() + .create_numbering(); + assert_eq!(actual, expected, "no synthetic (0, 0, 0) frame at FINISH"); + } + + #[test] + fn number_from_parts_keeps_a_one_frame_sizes_vector_on_the_default_path() { + // A single-frame sizes entry is the implicit layout `number` already + // emits. Using it as an explicit split would drop snapshot boxes + // past that count. + let snapshot = Snapshot { + framestack: vec![SnapshotFrame { + jitcode_index: 3, + pc: 4, + py_pc: 5, + boxes: vec![ + SnapshotBox::from(OpRef::ref_op(0)), + SnapshotBox::from(OpRef::const_int(9)), + SnapshotBox::from(OpRef::ref_op(1)), + ], + }], + vable_array: Vec::new(), + vref_array: Vec::new(), + }; + let env = SimpleBoxEnv::new(); + let expected = ResumeDataLoopMemo::new() + .number(&snapshot, &env, -1) + .unwrap() + .create_numbering(); + let boxes = snapshot.framestack[0].boxes.clone(); + let short_size = Some(&[1usize][..]); + let actual = ResumeDataLoopMemo::new() + .number_from_parts(&boxes, short_size, &[(3, 4, 5)], &[], &[], &env, -1) + .unwrap() + .create_numbering(); + assert_eq!( + actual, expected, + "a one-frame sizes vector must not truncate snapshot boxes" + ); + } + #[test] fn test_number_rebuild_roundtrip() { use majit_ir::OpRef; @@ -6113,26 +6179,12 @@ mod tests { /// census established the route exists; this establishes only that its /// observable precondition did not occur in this corpus. /// - /// `override=yes` on 3298/3298, so the assert below is evaluated on every - /// one of them — it is exercised, not merely present. - /// - /// `consume_vable_info` now asserts the identity is not `NULLREF` when an - /// override is supplied, so the aliasing encoding is refused rather than - /// resolved to a wrong slot. - /// - /// Run against the landed assert: the subject arm trips it in - /// `consume_vable_info` rather than reaching the `0xABCD` read. So the - /// refusal is on the path this encoding builds — reached and load-bearing, - /// which a compile cannot establish — and the outcome splits cleanly in - /// two. [`a_null_ref_register_is_seeded_when_no_identity_override_is_supplied`] - /// pins that the guard stays silent without an override; - /// [`a_null_ref_identity_is_refused_when_an_override_is_supplied`] pins that - /// it fires with one. - /// - /// Split rather than converted to one `#[should_panic]`: that attribute is - /// satisfied by EITHER arm panicking, so the control — the half proving the - /// guard does not over-fire — could die of an unrelated cause with the test - /// still green, under a name that still claims both directions. + /// `consume_vable_info` uses the override as the virtualizable object + /// even when the identity item is `NULLREF`, and leaves register matching + /// off so a null ref register stays null. The two tests split those + /// outcomes: [`a_null_ref_register_is_seeded_when_no_identity_override_is_supplied`] + /// pins the no-override decode; [`a_null_ref_identity_uses_the_override_without_aliasing_null_registers`] + /// pins that the override restores the object and does not rewrite r0. fn leaf3_resume_null_identity(identity_override: Option) -> (i64, i64) { use crate::blackhole::BlackholeInterpBuilder; use crate::jitcode::JitCodeBuilder; @@ -6208,18 +6260,17 @@ mod tests { } /// SUBJECT. `r0` holds `NULLREF`, the same spelling the folded-out identity - /// uses, so matching the override by tag would hand a live frame pointer to - /// a register the program left empty. `consume_vable_info` refuses the - /// encoding instead. - /// - /// `expected` is load-bearing: a bare `#[should_panic]` is satisfied by ANY - /// panic, including one from a fixture that stopped building the encoding - /// correctly and died in setup. The substring pins WHICH refusal fired. + /// uses. The override restores the virtualizable object; matching by tag + /// would also rewrite r0, so register matching stays off. #[test] - #[should_panic(expected = "virtualizable identity encoded as NULLREF")] - fn a_null_ref_identity_is_refused_when_an_override_is_supplied() { + fn a_null_ref_identity_uses_the_override_without_aliasing_null_registers() { const OVERRIDE: i64 = 0xABCD; - leaf3_resume_null_identity(Some(OVERRIDE)); + let (virtualizable_ptr, r0) = leaf3_resume_null_identity(Some(OVERRIDE)); + assert_eq!( + virtualizable_ptr, OVERRIDE, + "NULLREF identity uses the override object" + ); + assert_eq!(r0, 0, "null ref register is not rewritten to the override"); } /// resume.py `_prepare_virtuals` resets `virtuals_cache` to zeros. @@ -8168,7 +8219,19 @@ impl<'a> ResumeDataDirectReader<'a> { // never a trace-time pointer or an unrelated deadframe slot. let tagged_identity = self.resumecodereader.next_item() as i16; let encoded_identity = self.decode_ref(tagged_identity); - let virtualizable = identity_override.unwrap_or(encoded_identity); + // Override is only the recovery for an empty identity (`NULLREF` / + // unread failarg). A live TAGBOX is the virtualizable of *this* + // jitcode — an inlined callee's own PyFrame, not the portal. Using + // a caller-supplied portal/JITFRAME here restores the callee's + // payload onto the wrong object (`get_total_size` then disagrees + // with `vable_size - 1`). State-field hosts fold `&state` out of + // failargs, so they encode `NULLREF` and still take the override. + let encoded_empty = tagged_eq(tagged_identity, NULLREF) || encoded_identity == 0; + let virtualizable = if encoded_empty { + identity_override.unwrap_or(encoded_identity) + } else { + encoded_identity + }; // MAJIT_LEAF3_PROV census. The assert below refuses ONE value on ONE // arm; this reports the whole distribution on every call, because // "NULLREF cannot occur here" and "NULLREF was not observed here" are @@ -8200,36 +8263,19 @@ impl<'a> ResumeDataDirectReader<'a> { }, ); } - if identity_override.is_some() { - // `next_ref_for_resume_slot` routes a ref register to the override - // by comparing its tag against `virtualizable_identity_tagged`. That - // is only an identity test while the identity's tag is unique to it: - // `NULLREF` is the shared "no box here" encoding, so an identity - // recorded as `NULLREF` would claim every null ref register in the - // frame and hand each one the virtualizable pointer. - // - // `_number_boxes` emits `NULLREF` for a snapshot box whose `OpRef` - // is `NONE`, and `TreeLoop::cut_trace_from_with_consts` maps an - // unmapped pre-cut ref to `NONE` over `vable_boxes` as well as the - // frame sections. Its seed-or-cancel guard walks the snapshots of - // the post-cut ops, so a snapshot no post-cut op names is remapped - // without ever being able to cancel the compilation — the two loops - // iterate different populations. Refuse the aliasing encoding here - // rather than resolve a wrong slot silently. - // - // This BOUNDS the defect rather than closing it. It refuses the - // identity encoding it can name; the unseeded-snapshot case just - // described, which is the only remaining route, is still defaulted - // rather than refused. `NONE` there is a well-formed value standing - // in for an answer nobody computed, and that is precisely why - // nothing downstream can catch it — it is indistinguishable from - // the `NONE` that legitimately encodes a genuinely absent box. - assert!( - !tagged_eq(tagged_identity, NULLREF), - "virtualizable identity encoded as NULLREF while an identity \ - override is supplied: the tag is shared with every null ref \ - register, so the override cannot be matched to one slot" - ); + // `next_ref_for_resume_slot` routes a ref register to the override by + // comparing its tag against `virtualizable_identity_tagged`. That is + // only an identity test while the identity's tag is unique to it: + // `NULLREF` is the shared "no box here" encoding, so matching it would + // claim every null ref register and hand each one the virtualizable. + // + // `_number_boxes` emits `NULLREF` for a snapshot box whose `OpRef` is + // `NONE`, and `TreeLoop::cut_trace_from_with_consts` maps an unmapped + // pre-cut ref to `NONE` over `vable_boxes` as well as the frame + // sections. The host still knows the live virtualizable (portal + // PyFrame, state-field `&state`) and uses it as the object. + // Register matching stays off so empty refs stay empty. + if identity_override.is_some() && !tagged_eq(tagged_identity, NULLREF) { self.virtualizable_identity_tagged = Some(tagged_identity); self.virtualizable_identity_override = identity_override; } else { @@ -8258,6 +8304,17 @@ impl<'a> ResumeDataDirectReader<'a> { } } vinfo.push_resume_ref_roots(self.virtualizable_ptr); + // A cut remapped the identity to NONE (`NULLREF`). The remaining + // vable items are still in the stream and must be skipped so the + // vref/frame sections stay aligned, but their tags are not a typed + // field image — `pycode` has been observed as TAGINT on this path. + // The live virtualizable already holds the heap fields; writing the + // remapped payload would clobber them. + if tagged_eq(tagged_identity, NULLREF) { + self.resumecodereader.jump((vable_size - 1) as usize); + vinfo.reset_token_gcref(self.virtualizable_ptr); + return; + } // resume.py:1406: assert vinfo.get_total_size(virtualizable) == vable_size - 1 let expected = vinfo.get_total_size(self.virtualizable_ptr) as i32; assert!( @@ -8940,8 +8997,9 @@ pub fn blackhole_from_resumedata<'a>( /// Used for GUARD_NOT_FORCED handling. /// /// Returns (virtuals_cache_ptr, virtuals_cache_int) — RPython VirtualCache -/// parity — plus the virtualizable the vable section named, which the caller -/// needs as the cache key (see `MetaInterp::save_forced_virtuals`). +/// parity — plus the virtualizable the vable section named, matching the +/// upstream force decoder's result even though deadframe-owned `AllVirtuals` +/// no longer needs an address key. #[allow(clippy::needless_lifetimes)] #[expect( clippy::too_many_arguments, @@ -8959,6 +9017,7 @@ pub fn force_from_resumedata<'a>( vrefinfo: Option<&dyn VRefInfo>, vinfo: Option<&dyn VirtualizableInfo>, ginfo: Option<&dyn GreenfieldInfo>, + identity_override: Option, allocator: &'a dyn BlackholeAllocator, ) -> (Vec, Vec, i64) { let _bh_phase = majit_gc::BhProbePhase::enter("resume"); @@ -8987,7 +9046,7 @@ pub fn force_from_resumedata<'a>( prepare_resume_heap_with_roots(&mut resumereader, rd_virtuals, rd_guard_pendingfields); resumereader.handling_async_forcing(); // resume.py:1350 - resumereader.consume_vref_and_vable(vrefinfo, vinfo, ginfo, None); + resumereader.consume_vref_and_vable(vrefinfo, vinfo, ginfo, identity_override); // resume.py:1404 the virtualizable the vable section just named. The scope // above roots `virtuals_ptr_cache` only, not this field, so nothing would // forward the reader's slot in place — read it before `force_all_virtuals` diff --git a/majit/majit-metainterp/src/trace_ctx.rs b/majit/majit-metainterp/src/trace_ctx.rs index 60a99392d58..9c42a764e79 100644 --- a/majit/majit-metainterp/src/trace_ctx.rs +++ b/majit/majit-metainterp/src/trace_ctx.rs @@ -291,6 +291,21 @@ pub struct TraceCtx { /// (RPython parity: `virtualizable_boxes[-1]`). Used by gen_store_back_in_vable /// to distinguish standard vs nonstandard virtualizable. pub(crate) virtualizable_boxes: Option>, + /// Snapshot of `virtualizable_boxes` at `init_virtualizable_boxes` / + /// `set_virtualizable_boxes_with_info`. `gen_store_back_in_vable` skips + /// a static field or array slot whose live box is still this opref — + /// the `xxx only write back the fields really modified` note on + /// `pyjitpl.py gen_store_back_in_vable`. `replace_box` updates the live + /// list only, so a renamed box is still written back. + virtualizable_boxes_at_entry: Option>, + /// Mint field indices of GET_ITER iterables recorded while tracing. + /// leftover-empty binds those leftovers to this field even when + /// virtualstate replaced the mint box with a new InputArg. + pub(crate) getiter_vable_fields: Vec, + /// Inlined callee's own red frame (`emit_new_pyframe_inline`). leftover-empty + /// GETFIELDs this instead of the portal red when the leftover iterator + /// belongs to the callee (`_compile` inlined into its caller). + pub(crate) inline_vable_box: Option, /// Concrete shadow of `virtualizable_boxes`. Same layout, each slot carries /// the current runtime `Value` (RPython Box ≡ OpRef + concrete value). /// Seeded from `original_boxes` in `initialize_virtualizable` and kept in @@ -1805,6 +1820,9 @@ impl TraceCtx { green_key_values: None, driver_descriptor: None, virtualizable_boxes: None, + virtualizable_boxes_at_entry: None, + getiter_vable_fields: Vec::new(), + inline_vable_box: None, virtualizable_values: None, virtualizable_live_null_slots: None, virtualizable_info: None, @@ -1905,6 +1923,9 @@ impl TraceCtx { green_key_values: Some(green_key_values), driver_descriptor: None, virtualizable_boxes: None, + virtualizable_boxes_at_entry: None, + getiter_vable_fields: Vec::new(), + inline_vable_box: None, virtualizable_values: None, virtualizable_live_null_slots: None, virtualizable_info: None, @@ -2595,6 +2616,46 @@ impl TraceCtx { /// are the OpRef and concrete of the virtualizable object (frame pointer). /// Boxes layout: `[field0, ..., fieldN, arr[0], ..., arr[M], vable_ref]` /// where `boxes[-1]` is the standard virtualizable identity (RPython parity). + /// If `args` of a GET_ITER residual include a virtualizable field box, + /// remember that mint index for leftover-empty. + pub fn note_getiter_iterable(&mut self, args: &[OpRef]) { + let field_len = self + .virtualizable_boxes_at_entry + .as_ref() + .or(self.virtualizable_boxes.as_ref()) + .map(|boxes| boxes.len().saturating_sub(1)) + .unwrap_or(0); + if field_len == 0 { + return; + } + let match_in = |boxes: &[OpRef], arg: OpRef| -> Option { + boxes[..field_len.min(boxes.len())] + .iter() + .position(|&m| m == arg) + .map(|j| j as u32) + }; + for &arg in args { + if arg.is_none() || arg.is_constant() { + continue; + } + let found = self + .virtualizable_boxes_at_entry + .as_ref() + .and_then(|boxes| match_in(boxes, arg)) + .or_else(|| { + self.virtualizable_boxes + .as_ref() + .and_then(|boxes| match_in(boxes, arg)) + }); + if let Some(j) = found { + if !self.getiter_vable_fields.contains(&j) { + self.getiter_vable_fields.push(j); + } + return; + } + } + } + pub fn init_virtualizable_boxes( &mut self, info: &VirtualizableInfo, @@ -2606,6 +2667,7 @@ impl TraceCtx { ) { let mut boxes = input_oprefs.to_vec(); boxes.push(vable_ref); // RPython: virtualizable_boxes[-1] = vable identity + self.virtualizable_boxes_at_entry = Some(boxes.clone()); self.virtualizable_boxes = Some(boxes); if input_values.is_empty() { // Caller has no live concrete values (e.g. bridge-entry rebuild @@ -3540,6 +3602,49 @@ impl TraceCtx { .is_some_and(|slots| slots.get(index).copied().unwrap_or(false)) } + pub fn set_inline_vable_box(&mut self, frame: OpRef) { + if !frame.is_none() { + self.inline_vable_box = Some(frame); + } + } + + pub fn inline_vable_box(&self) -> Option { + self.inline_vable_box + } + + /// The IR box whose concrete is `addr`, if it is not the portal red. + /// leftover-empty GETFIELDs this inlined `_compile` frame instead of + /// `inputargs[index_of_virtualizable]`. + pub fn opref_with_concrete_ref(&self, addr: usize, skip: OpRef) -> Option { + if addr == 0 { + return None; + } + let is_addr = |opref: OpRef| -> bool { + matches!( + self.concrete_of_opref(opref), + Some(Value::Ref(r)) if r.as_usize() == addr + ) + }; + if let Some(b) = self.standard_virtualizable_box() { + if b != skip && is_addr(b) { + return Some(b); + } + } + for op in self.recorder.ops() { + let pos = op.pos().get(); + if !pos.is_none() && pos != skip && is_addr(pos) { + return Some(pos); + } + for a in op.getarglist() { + let r = a.to_opref(); + if r != skip && !r.is_none() && is_addr(r) { + return Some(r); + } + } + } + None + } + /// Return the standard virtualizable identity (`virtualizable_boxes[-1]`). pub fn standard_virtualizable_box(&self) -> Option { self.virtualizable_boxes @@ -3769,6 +3874,7 @@ impl TraceCtx { /// resume data before the bridge replays any vable op. pub fn clear_virtualizable_boxes(&mut self) { self.virtualizable_boxes = None; + self.virtualizable_boxes_at_entry = None; } /// Set virtualizable_boxes with VirtualizableInfo and array lengths. @@ -3799,6 +3905,7 @@ impl TraceCtx { self.virtualizable_values = None; self.virtualizable_live_null_slots = None; } + self.virtualizable_boxes_at_entry = Some(boxes.clone()); self.virtualizable_boxes = Some(boxes); self.virtualizable_info = Some(std::sync::Arc::new(info.clone())); self.virtualizable_array_lengths = Some(array_lengths.to_vec()); @@ -4029,9 +4136,34 @@ impl TraceCtx { // pyjitpl.py:3478 self.forced_virtualizable = vbox self.forced_virtualizable = Some(vable_opref); + // pyjitpl.py `xxx only write back the fields really modified`. + // A slot whose live box is still the entry snapshot is the heap + // value `initialize_virtualizable` read; writing it is a no-op + // and, unlike upstream, OptHeap cannot cancel it: the portal + // fields ride as inputargs, not GETFIELD, so the lazy-set cache + // is empty. `replace_box` leaves the snapshot alone, so a + // renamed box is still stored. + let entry = self.virtualizable_boxes_at_entry.clone(); + let box_unchanged = |i: usize, value: OpRef| -> bool { + entry.as_deref().and_then(|e| e.get(i).copied()) == Some(value) + }; + for field_index in 0..info.static_fields.len() { if let Some(&value) = boxes.get(field_index) { - let descr = info.static_field_descr(field_index); + if box_unchanged(field_index, value) { + continue; + } + // pyjitpl.py `gen_store_back_in_vable` records SETFIELD_GC + // with `vinfo.static_field_descrs[i]`. Upstream that list + // is `cpu.fielddescrof(VTYPE, name)` — the same FieldDescr + // the interpreter's SETFIELD_GC uses (`descr.py`). Pyre + // splits the vable schedule (`index_in_parent` = + // `[token, statics, arrays]`) from the parent SizeDescr + // walker; OptHeap keys the lazy-set cache by descr + // identity, so the store-back must reuse the parent + // field (`static_field_struct_descr`) or last_instr is + // written twice and unchanged slots cannot cancel. + let descr = info.static_field_struct_descr(field_index); // pyjitpl.py `gen_store_back_in_vable`. A store has no // `resvalue` and `SETFIELD_GC` is never pure, so no cpu. self.execute_and_record( @@ -4048,22 +4180,34 @@ impl TraceCtx { let mut flat_box_index = info.static_fields.len(); for array_index in 0..info.array_fields.len() { let len = lengths.get(array_index).copied().unwrap_or(0); - let field_descr = info.array_pointer_field_descr(array_index); - let array_descr = info.array_item_descr(array_index); - let array_ref = self.vable_getfield_ref_descr(vable_opref, field_descr); - for item_index in 0..len { - if let Some(&value) = boxes.get(flat_box_index) { - let index = self.const_int(item_index as i64); - self.execute_and_record( - None, - OpCode::SetarrayitemGc, - Some(array_descr.clone()), - &[array_ref, index, value], - None, - 0, - ); + let array_start = flat_box_index; + let any_item_changed = (0..len).any(|item_index| { + boxes + .get(array_start + item_index) + .is_some_and(|&value| !box_unchanged(array_start + item_index, value)) + }); + if any_item_changed { + let field_descr = info.array_pointer_struct_descr(array_index); + let array_descr = info.array_item_descr(array_index); + let array_ref = self.vable_getfield_ref_descr(vable_opref, field_descr); + for item_index in 0..len { + if let Some(&value) = boxes.get(flat_box_index) + && !box_unchanged(flat_box_index, value) + { + let index = self.const_int(item_index as i64); + self.execute_and_record( + None, + OpCode::SetarrayitemGc, + Some(array_descr.clone()), + &[array_ref, index, value], + None, + 0, + ); + } + flat_box_index += 1; } - flat_box_index += 1; + } else { + flat_box_index += len; } } @@ -6333,6 +6477,20 @@ mod tests { ); } + #[test] + fn inline_vable_box_is_the_seeded_callee_frame() { + let mut ctx = TraceCtx::for_test(1); + let callee = OpRef::ref_op(7); + ctx.set_inline_vable_box(callee); + assert_eq!(ctx.inline_vable_box(), Some(callee)); + ctx.set_inline_vable_box(OpRef::NONE); + assert_eq!( + ctx.inline_vable_box(), + Some(callee), + "NONE must not clear the seeded callee red" + ); + } + /// `heapcache.py is_nullity_known` answers truthy for a non-`Const` box /// whatever its nullity — `nullity_now_known` sets one flag for both — and /// falsy for a null `Const`, whose answer is `bool(box.getref_base())`. @@ -6878,6 +7036,108 @@ mod tests { ); } + #[test] + fn test_record_getiter_remembers_vable_field_index() { + let mut info = crate::virtualizable::VirtualizableInfo::new(0); + info.add_field("last_instr", Type::Int, 8); + info.add_field("pycode", Type::Ref, 16); + info.add_field("valuestackdepth", Type::Int, 24); + info.add_field("debugdata", Type::Ref, 32); + info.set_parent_descr(majit_ir::descr::make_size_descr(40)); + let mut recorder = Trace::new(); + let vable = recorder.record_input_arg(Type::Ref); + let last_instr = recorder.record_input_arg(Type::Int); + let pycode = recorder.record_input_arg(Type::Ref); + let depth = recorder.record_input_arg(Type::Int); + let debugdata = recorder.record_input_arg(Type::Ref); + let pattern = recorder.record_input_arg(Type::Ref); + let mut ctx = TraceCtx::new( + recorder, + 0, + std::sync::Arc::new(crate::MetaInterpStaticData::new()), + ); + ctx.init_virtualizable_boxes( + &info, + vable, + ph(Type::Ref), + &[last_instr, pycode, depth, debugdata, pattern], + &[ + ph(Type::Int), + ph(Type::Ref), + ph(Type::Int), + ph(Type::Ref), + ph(Type::Ref), + ], + &[], + ); + let mut effect = majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CanRaise, + majit_ir::OopSpecIndex::None, + ); + effect.runtime_helper = majit_ir::RuntimeHelperKind::GetIter; + let descr = majit_ir::descr::make_call_descr(vec![Type::Ref], Type::Ref, effect); + ctx.record_op_with_descr(OpCode::CallR, &[pattern], descr); + assert_eq!( + ctx.getiter_vable_fields, + vec![4], + "GET_ITER of the pattern mint box must record field 4" + ); + } + + #[test] + fn test_record_listiter_seq_getfield_remembers_vable_field_index() { + // pip `_compile` traces FOR_ITER as Getfield of W_ListIterObject.seq, + // not RuntimeHelperKind::GetIter. The receiver is the iterator mint. + let mut info = crate::virtualizable::VirtualizableInfo::new(0); + info.add_field("last_instr", Type::Int, 8); + info.add_field("pycode", Type::Ref, 16); + info.add_field("valuestackdepth", Type::Int, 24); + info.add_field("debugdata", Type::Ref, 32); + info.set_parent_descr(majit_ir::descr::make_size_descr(40)); + let mut recorder = Trace::new(); + let vable = recorder.record_input_arg(Type::Ref); + let last_instr = recorder.record_input_arg(Type::Int); + let pycode = recorder.record_input_arg(Type::Ref); + let depth = recorder.record_input_arg(Type::Int); + let debugdata = recorder.record_input_arg(Type::Ref); + let listiter = recorder.record_input_arg(Type::Ref); + let mut ctx = TraceCtx::new( + recorder, + 0, + std::sync::Arc::new(crate::MetaInterpStaticData::new()), + ); + ctx.init_virtualizable_boxes( + &info, + vable, + ph(Type::Ref), + &[last_instr, pycode, depth, debugdata, listiter], + &[ + ph(Type::Int), + ph(Type::Ref), + ph(Type::Int), + ph(Type::Ref), + ph(Type::Ref), + ], + &[], + ); + let descr = std::sync::Arc::new(majit_ir::descr::SimpleFieldDescr::new_with_name( + 0, + 16, + 8, + Type::Ref, + false, + majit_ir::ArrayFlag::Pointer, + "W_ListIterObject.seq".into(), + "seq".into(), + )) as majit_ir::DescrRef; + ctx.record_op_with_descr(OpCode::GetfieldGcR, &[listiter], descr); + assert_eq!( + ctx.getiter_vable_fields, + vec![4], + "Getfield of ListIter.seq must record the iterator mint field" + ); + } + #[test] fn standard_vable_setfield_writes_to_boxes() { let info = make_test_vable_info(); @@ -7323,33 +7583,77 @@ mod tests { &[2], ); + let new_pc = ctx.const_int(7); + let new_arr1 = ctx.const_ref(99); + if let Some(boxes) = ctx.virtualizable_boxes.as_mut() { + boxes[0] = new_pc; + boxes[2] = new_arr1; + } ctx.gen_store_back_in_vable(vable); let ops = take_all_ops(ctx); - assert_eq!(ops.len(), 5); + assert_eq!(ops.len(), 4); assert_eq!(ops[0].opcode, OpCode::SetfieldGc); assert_eq!( ops[0].getdescr().map(|d| d.index()), - Some(info.static_field_descr(0).index()) + Some(info.static_field_struct_descr(0).index()) ); assert_eq!(ops[1].opcode, OpCode::GetfieldGcR); assert_eq!( ops[1].getdescr().map(|d| d.index()), - Some(info.array_pointer_field_descr(0).index()) + Some(info.array_pointer_struct_descr(0).index()) ); assert_eq!(ops[2].opcode, OpCode::SetarrayitemGc); assert_eq!( ops[2].getdescr().map(|d| d.index()), Some(info.array_item_descr(0).index()) ); - assert_eq!(ops[3].opcode, OpCode::SetarrayitemGc); + assert_eq!(ops[3].opcode, OpCode::SetfieldGc); assert_eq!( ops[3].getdescr().map(|d| d.index()), - Some(info.array_item_descr(0).index()) + Some(info.token_field_descr().index()) ); - assert_eq!(ops[4].opcode, OpCode::SetfieldGc); + } + + #[test] + fn gen_store_back_in_vable_skips_unmodified_fields() { + let mut info = crate::virtualizable::VirtualizableInfo::new(0); + info.add_field("pc", Type::Int, 8); + info.add_array_field( + "locals", + Type::Ref, + 24, + 0, + 0, + majit_ir::make_array_descr(0, 8, Type::Ref), + ); + info.set_parent_descr(majit_ir::descr::make_size_descr(64)); + + let mut recorder = Trace::new(); + let vable = recorder.record_input_arg(Type::Ref); + let box_pc = recorder.record_input_arg(Type::Int); + let box_arr0 = recorder.record_input_arg(Type::Ref); + let mut ctx = TraceCtx::new( + recorder, + 0, + std::sync::Arc::new(crate::MetaInterpStaticData::new()), + ); + ctx.init_virtualizable_boxes( + &info, + vable, + ph(Type::Ref), + &[box_pc, box_arr0], + &[ph(Type::Int), ph(Type::Ref)], + &[1], + ); + + ctx.gen_store_back_in_vable(vable); + + let ops = take_all_ops(ctx); + assert_eq!(ops.len(), 1); + assert_eq!(ops[0].opcode, OpCode::SetfieldGc); assert_eq!( - ops[4].getdescr().map(|d| d.index()), + ops[0].getdescr().map(|d| d.index()), Some(info.token_field_descr().index()) ); } diff --git a/majit/majit-rlib/src/lltypesystem/rlist.rs b/majit/majit-rlib/src/lltypesystem/rlist.rs index 0a7c41207ad..4932e018064 100644 --- a/majit/majit-rlib/src/lltypesystem/rlist.rs +++ b/majit/majit-rlib/src/lltypesystem/rlist.rs @@ -293,6 +293,49 @@ pub unsafe fn try_alloc_typed_items_block_nursery( } } +/// Collecting `malloc_fast` for an items block with the translated graph's +/// complete native live-root span. This is the Rust ABI of +/// `gct_fv_gc_malloc`'s `push_roots(hop)` / `pop_roots(hop)` bracket; MiniMark +/// consults the span only when the nursery bump reaches +/// `collect_and_reserve`. +/// +/// # Safety +/// Every entry in `roots` must be a valid GC object pointer or null. The +/// slice must remain address-stable for this call and callers must reload its +/// entries after return before dereferencing any pre-call pointer copies. +pub unsafe fn try_alloc_typed_items_block_nursery_rooted( + cap: usize, + tid: u32, + roots: &mut [majit_ir::GcRef], +) -> Option<*mut TypedItemsBlock> { + let cap = cap.max(1); + let layout = try_typed_items_block_layout(cap)?; + if itemsblock_gc_enabled() && majit_gc::gc_allocator_installed() { + assert!( + tid != UNSET_GC_TYPE_ID, + "items block allocated with an undeclared GC type id" + ); + let mut needs_write_barrier = false; + let raw = unsafe { + majit_gc::alloc_fast_nursery_collecting_typed_roots( + tid, + layout.size(), + roots.as_mut_ptr(), + roots.len(), + &mut needs_write_barrier, + ) + } + .0 as *mut u8; + if raw.is_null() { + return None; + } + let block = raw as *mut TypedItemsBlock; + unsafe { (*block).capacity = cap }; + return Some(block); + } + unsafe { try_alloc_typed_items_block_nursery(cap, tid) } +} + /// `rgc.ll_arrayclear(l.ll_items())` — zero every item of a freshly allocated /// block, the second half of `ll_alloc_and_set(LIST, count, 0)` /// (rtyper/rlist.py:494-503). @@ -384,6 +427,32 @@ impl Digits { } } + /// `ll_alloc_and_set(..., 0)` at a malloc whose transformed graph carries + /// more than one live GC variable. Kept beside the ordinary list + /// operation because the root span is allocation machinery, not bigint + /// arithmetic. + /// + /// # Safety + /// See [`try_alloc_typed_items_block_nursery_rooted`]. + #[inline] + pub unsafe fn alloc_and_set_zero_rooted( + count: usize, + roots: &mut [majit_ir::GcRef], + ) -> *mut TypedItemsBlock { + unsafe { + let block = + try_alloc_typed_items_block_nursery_rooted(count, gc_int_array_gc_type_id(), roots) + .unwrap_or_else(|| { + std::alloc::handle_alloc_error( + try_typed_items_block_layout(count.max(1)) + .unwrap_or_else(|| Layout::new::()), + ) + }); + typed_items_block_clear(block); + block + } + } + /// The one-element body of a prebuilt list, at process lifetime. /// /// Upstream's prebuilt `rbigint` digit lists are translated constants with diff --git a/majit/majit-rlib/src/rbigint.rs b/majit/majit-rlib/src/rbigint.rs index ba7828e874c..a09093cd77a 100644 --- a/majit/majit-rlib/src/rbigint.rs +++ b/majit/majit-rlib/src/rbigint.rs @@ -661,6 +661,33 @@ impl RBigInt { } } + /// `[NULLDIGIT] * size` with the GC-transform livevars that + /// `gct_fv_gc_malloc` publishes across that malloc. Reloads `a` and `b` + /// from the root span after a moving collection. + fn with_size_reloading_operands<'a>( + size: i64, + sign: i64, + a: &mut &'a RBigInt, + b: &mut &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, + ) -> Self { + debug_assert!(size >= 0); + if let Some(roots) = digit_roots { + debug_assert!(roots.len() >= 2); + roots[0] = majit_ir::GcRef((*a as *const RBigInt) as usize); + roots[1] = majit_ir::GcRef((*b as *const RBigInt) as usize); + let block = unsafe { Digits::alloc_and_set_zero_rooted(size as usize, roots) }; + *a = unsafe { &*(roots[0].0 as *const RBigInt) }; + *b = unsafe { &*(roots[1].0 as *const RBigInt) }; + Self { + _digits: block, + _size: size * sign, + } + } else { + Self::with_size(size, sign) + } + } + /// The explicit Rust form of RPython's implicit `gc_malloc_array` /// `MemoryError` edge. Used by operations whose public Rust signature /// already carries `RBigIntError`. @@ -1328,6 +1355,14 @@ impl RBigInt { #[majit_macros::jit_elidable] pub fn add(&self, other: &Self) -> Self { + self.add_with_gc_roots(other, None) + } + + /// The body of `rbigint.add`, with the optional live-root vector inserted + /// by RPython's GC transform around the digit-list malloc. Ordinary Rust + /// callers take the source-shaped `None` arm; GC-payload residuals supply + /// the transformed graph's roots through `rbigint::gc`. + fn add_with_gc_roots(&self, other: &Self, digit_roots: Option<&mut [majit_ir::GcRef]>) -> Self { let selfsign = self.get_sign(); let othersign = other.get_sign(); if selfsign == 0 { @@ -1337,9 +1372,9 @@ impl RBigInt { return self.translated_alias(); } let mut result = if selfsign == othersign { - _x_add(self, other) + _x_add(self, other, digit_roots) } else { - _x_sub(other, self) + _x_sub(other, self, digit_roots) }; result._set_sign(result.get_sign() * othersign); result @@ -1416,6 +1451,11 @@ impl RBigInt { #[majit_macros::jit_elidable] pub fn sub(&self, other: &Self) -> Self { + self.sub_with_gc_roots(other, None) + } + + /// GC-transformed twin of `sub`, for the residual payload boundary. + fn sub_with_gc_roots(&self, other: &Self, digit_roots: Option<&mut [majit_ir::GcRef]>) -> Self { let selfsign = self.get_sign(); let othersign = other.get_sign(); if othersign == 0 { @@ -1429,9 +1469,9 @@ impl RBigInt { ); } let mut result = if selfsign == othersign { - _x_sub(self, other) + _x_sub(self, other, digit_roots) } else { - _x_add(self, other) + _x_add(self, other, digit_roots) }; result._set_sign(result.get_sign() * selfsign); result @@ -1468,6 +1508,11 @@ impl RBigInt { #[majit_macros::jit_elidable] pub fn mul(&self, other: &Self) -> Self { + self.mul_with_gc_roots(other, None) + } + + /// GC-transformed twin of `mul`, for the residual payload boundary. + fn mul_with_gc_roots(&self, other: &Self, digit_roots: Option<&mut [majit_ir::GcRef]>) -> Self { let mut this = self; let mut that = other; let mut selfsize = this.numdigits(); @@ -1510,7 +1555,7 @@ impl RBigInt { 1, ); } - result = _x_mul(this, that, this.digit(0)); + result = _x_mul(this, that, this.digit(0), digit_roots); } else if USE_KARATSUBA { let cutoff = if std::ptr::eq(this, that) { KARATSUBA_SQUARE_CUTOFF @@ -1518,12 +1563,12 @@ impl RBigInt { KARATSUBA_CUTOFF }; if selfsize <= cutoff { - result = _x_mul(this, that, 0); + result = _x_mul(this, that, 0, digit_roots); } else { - result = _k_mul(this, that); + result = _k_mul(this, that, digit_roots); } } else { - result = _x_mul(this, that, 0); + result = _x_mul(this, that, 0, digit_roots); } result._set_sign(selfsign * othersign); result @@ -2296,7 +2341,11 @@ impl RBigInt { #[majit_macros::jit_elidable] pub fn and_(&self, other: &Self) -> Self { - _bitwise_and(self, other) + self.and_with_gc_roots(other, None) + } + + fn and_with_gc_roots(&self, other: &Self, digit_roots: Option<&mut [majit_ir::GcRef]>) -> Self { + _bitwise_and(self, other, digit_roots) } #[majit_macros::jit_elidable] @@ -2306,7 +2355,11 @@ impl RBigInt { #[majit_macros::jit_elidable] pub fn xor(&self, other: &Self) -> Self { - _bitwise_xor(self, other) + self.xor_with_gc_roots(other, None) + } + + fn xor_with_gc_roots(&self, other: &Self, digit_roots: Option<&mut [majit_ir::GcRef]>) -> Self { + _bitwise_xor(self, other, digit_roots) } #[majit_macros::jit_elidable] @@ -2316,7 +2369,11 @@ impl RBigInt { #[majit_macros::jit_elidable] pub fn or_(&self, other: &Self) -> Self { - _bitwise_or(self, other) + self.or_with_gc_roots(other, None) + } + + fn or_with_gc_roots(&self, other: &Self, digit_roots: Option<&mut [majit_ir::GcRef]>) -> Self { + _bitwise_or(self, other, digit_roots) } #[majit_macros::jit_elidable] @@ -3117,14 +3174,18 @@ fn args_from_long(value: i128) -> (Vec, i64) { } /// rbigint.py `_x_add`. -fn _x_add<'a>(mut a: &'a RBigInt, mut b: &'a RBigInt) -> RBigInt { +fn _x_add<'a>( + mut a: &'a RBigInt, + mut b: &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { let mut size_a = a.numdigits(); let mut size_b = b.numdigits(); if size_a < size_b { std::mem::swap(&mut a, &mut b); std::mem::swap(&mut size_a, &mut size_b); } - let mut z = RBigInt::with_size(size_a + 1, 1); + let mut z = RBigInt::with_size_reloading_operands(size_a + 1, 1, &mut a, &mut b, digit_roots); let mut i = 0; let mut carry = 0_u64; while i < size_b { @@ -3164,7 +3225,11 @@ fn _x_int_add(a: &RBigInt, b: i64) -> RBigInt { } /// rbigint.py `_x_sub`. -fn _x_sub<'a>(mut a: &'a RBigInt, mut b: &'a RBigInt) -> RBigInt { +fn _x_sub<'a>( + mut a: &'a RBigInt, + mut b: &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { let mut size_a = a.numdigits(); let mut size_b = b.numdigits(); let mut sign = 1; @@ -3188,7 +3253,7 @@ fn _x_sub<'a>(mut a: &'a RBigInt, mut b: &'a RBigInt) -> RBigInt { size_b = i; } - let mut z = RBigInt::with_size(size_a, sign); + let mut z = RBigInt::with_size_reloading_operands(size_a, sign, &mut a, &mut b, digit_roots); let mut borrow = 0_u64; let mut i = 0; while i < size_b { @@ -3260,12 +3325,18 @@ const fn make_ptwotable() -> [i64; SHIFT as usize] { const PTWOTABLE: [i64; SHIFT as usize] = make_ptwotable(); /// rbigint.py `_x_mul`. -fn _x_mul(a: &RBigInt, b: &RBigInt, digit: Digit) -> RBigInt { +fn _x_mul<'a>( + mut a: &'a RBigInt, + mut b: &'a RBigInt, + digit: Digit, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { let size_a = a.numdigits(); let size_b = b.numdigits(); if std::ptr::eq(a, b) { - let mut z = RBigInt::with_size(size_a + size_b, 1); + let mut z = + RBigInt::with_size_reloading_operands(size_a + size_b, 1, &mut a, &mut b, digit_roots); let mut i = 0; while i < size_a { let mut f = a.uwidedigit(i); @@ -3305,7 +3376,8 @@ fn _x_mul(a: &RBigInt, b: &RBigInt, digit: Digit) -> RBigInt { return _muladd1(b, digit, 0); } - let mut z = RBigInt::with_size(size_a + size_b, 1); + let mut z = + RBigInt::with_size_reloading_operands(size_a + size_b, 1, &mut a, &mut b, digit_roots); let mut i = 0; let size_a1 = size_a - 1; let size_b1 = size_b - 1; @@ -3379,10 +3451,18 @@ fn _kmul_split(n: &RBigInt, size: i64) -> (RBigInt, RBigInt) { } /// rbigint.py `_k_mul`. -fn _k_mul(a: &RBigInt, b: &RBigInt) -> RBigInt { +fn _k_mul<'a>( + mut a: &'a RBigInt, + mut b: &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { let asize = a.numdigits(); let bsize = b.numdigits(); - let mut ret = RBigInt::with_size(asize + bsize, 1); + // Collecting malloc of the result buffer. Recursive `mul` below stays + // on the source `None` arm: those intermediates are fresh nursery + // objects whose `_digits` are not in this residual's livevar span. + let mut ret = + RBigInt::with_size_reloading_operands(asize + bsize, 1, &mut a, &mut b, digit_roots); let shift = bsize >> 1; let (bh, bl) = _kmul_split(b, shift); @@ -3431,11 +3511,11 @@ fn _k_mul(a: &RBigInt, b: &RBigInt) -> RBigInt { _v_isub(&mut ret, shift, i, &t2, t2.numdigits()); _v_isub(&mut ret, shift, i, &t1, t1.numdigits()); - let t1 = _x_add(ah, al); + let t1 = _x_add(ah, al, None); let t3 = if std::ptr::eq(a, b) { t1.mul(&t1) } else { - let t2 = _x_add(&bh, &bl); + let t2 = _x_add(&bh, &bl, None); t1.mul(&t2) }; debug_assert!(t3.get_sign() >= 0); @@ -4812,22 +4892,58 @@ fn _format( Ok(output) } -/// rbigint.py `@specialize.arg(1) _bitwise(a, '&', b)`. -fn _bitwise_and(a: &RBigInt, b: &RBigInt) -> RBigInt { - let a_inverted; - let (a, mut maska) = if a.get_sign() < 0 { - a_inverted = a.invert(); - (&a_inverted, MASK as Digit) +/// Holds an inverted operand so its `_digits` stay live across a collecting +/// `with_size`. The residual root span names the heap payloads, not these +/// stack handles; `RBigIntGcRoot` is the `_digits` shadow-stack slot. +struct BitwiseInvertHold { + root: Option, + hold: Option, +} + +fn bitwise_prep_operand<'a>( + value: &'a RBigInt, + collect: bool, + storage: &'a mut BitwiseInvertHold, +) -> (&'a RBigInt, Digit, bool) { + if value.get_sign() < 0 { + let inverted = value.invert(); + if collect { + storage.root = Some(RBigIntGcRoot::new(inverted)); + ( + storage.root.as_ref().expect("just stored"), + MASK as Digit, + true, + ) + } else { + storage.hold = Some(inverted); + ( + storage.hold.as_ref().expect("just stored"), + MASK as Digit, + true, + ) + } } else { - (a, 0) + (value, 0, false) + } +} + +/// rbigint.py `@specialize.arg(1) _bitwise(a, '&', b)`. +fn _bitwise_and<'a>( + orig_a: &'a RBigInt, + orig_b: &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { + let collect = digit_roots.is_some(); + let mut a_storage = BitwiseInvertHold { + root: None, + hold: None, }; - let b_inverted; - let (b, mut maskb) = if b.get_sign() < 0 { - b_inverted = b.invert(); - (&b_inverted, MASK as Digit) - } else { - (b, 0) + let mut b_storage = BitwiseInvertHold { + root: None, + hold: None, }; + let (a, mut maska, inverted_a) = bitwise_prep_operand(orig_a, collect, &mut a_storage); + let (b, mut maskb, inverted_b) = bitwise_prep_operand(orig_b, collect, &mut b_storage); let mut negz = false; let mut op_is_and = true; @@ -4851,7 +4967,12 @@ fn _bitwise_and(a: &RBigInt, b: &RBigInt) -> RBigInt { } else { size_a.max(size_b) }; - let mut z = RBigInt::with_size(size_z, 1); + let mut live_a = orig_a; + let mut live_b = orig_b; + let mut z = + RBigInt::with_size_reloading_operands(size_z, 1, &mut live_a, &mut live_b, digit_roots); + let a = if inverted_a { a } else { live_a }; + let b = if inverted_b { b } else { live_b }; let mut i = 0; while i < size_z { let diga = if i < size_a { @@ -4873,21 +4994,22 @@ fn _bitwise_and(a: &RBigInt, b: &RBigInt) -> RBigInt { } /// The `'|'` graph emitted by upstream's `@specialize.arg(1) _bitwise`. -fn _bitwise_or(a: &RBigInt, b: &RBigInt) -> RBigInt { - let a_inverted; - let (a, mut maska) = if a.get_sign() < 0 { - a_inverted = a.invert(); - (&a_inverted, MASK as Digit) - } else { - (a, 0) +fn _bitwise_or<'a>( + orig_a: &'a RBigInt, + orig_b: &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { + let collect = digit_roots.is_some(); + let mut a_storage = BitwiseInvertHold { + root: None, + hold: None, }; - let b_inverted; - let (b, mut maskb) = if b.get_sign() < 0 { - b_inverted = b.invert(); - (&b_inverted, MASK as Digit) - } else { - (b, 0) + let mut b_storage = BitwiseInvertHold { + root: None, + hold: None, }; + let (a, mut maska, inverted_a) = bitwise_prep_operand(orig_a, collect, &mut a_storage); + let (b, mut maskb, inverted_b) = bitwise_prep_operand(orig_b, collect, &mut b_storage); let mut negz = false; let mut op_is_and = false; @@ -4911,7 +5033,12 @@ fn _bitwise_or(a: &RBigInt, b: &RBigInt) -> RBigInt { } else { size_a.max(size_b) }; - let mut z = RBigInt::with_size(size_z, 1); + let mut live_a = orig_a; + let mut live_b = orig_b; + let mut z = + RBigInt::with_size_reloading_operands(size_z, 1, &mut live_a, &mut live_b, digit_roots); + let a = if inverted_a { a } else { live_a }; + let b = if inverted_b { b } else { live_b }; let mut i = 0; while i < size_z { let diga = if i < size_a { @@ -4933,21 +5060,22 @@ fn _bitwise_or(a: &RBigInt, b: &RBigInt) -> RBigInt { } /// The `'^'` graph emitted by upstream's `@specialize.arg(1) _bitwise`. -fn _bitwise_xor(a: &RBigInt, b: &RBigInt) -> RBigInt { - let a_inverted; - let (a, mut maska) = if a.get_sign() < 0 { - a_inverted = a.invert(); - (&a_inverted, MASK as Digit) - } else { - (a, 0) +fn _bitwise_xor<'a>( + orig_a: &'a RBigInt, + orig_b: &'a RBigInt, + digit_roots: Option<&mut [majit_ir::GcRef]>, +) -> RBigInt { + let collect = digit_roots.is_some(); + let mut a_storage = BitwiseInvertHold { + root: None, + hold: None, }; - let b_inverted; - let (b, maskb) = if b.get_sign() < 0 { - b_inverted = b.invert(); - (&b_inverted, MASK as Digit) - } else { - (b, 0) + let mut b_storage = BitwiseInvertHold { + root: None, + hold: None, }; + let (a, mut maska, inverted_a) = bitwise_prep_operand(orig_a, collect, &mut a_storage); + let (b, maskb, inverted_b) = bitwise_prep_operand(orig_b, collect, &mut b_storage); let mut negz = false; if maska != maskb { @@ -4958,7 +5086,12 @@ fn _bitwise_xor(a: &RBigInt, b: &RBigInt) -> RBigInt { let size_a = a.numdigits(); let size_b = b.numdigits(); let size_z = size_a.max(size_b); - let mut z = RBigInt::with_size(size_z, 1); + let mut live_a = orig_a; + let mut live_b = orig_b; + let mut z = + RBigInt::with_size_reloading_operands(size_z, 1, &mut live_a, &mut live_b, digit_roots); + let a = if inverted_a { a } else { live_a }; + let b = if inverted_b { b } else { live_b }; let mut i = 0; while i < size_z { let diga = if i < size_a { @@ -4982,7 +5115,7 @@ fn _bitwise_xor(a: &RBigInt, b: &RBigInt) -> RBigInt { /// `@specialize.arg(1) _int_bitwise(a, '&', b)`. fn _int_bitwise_and(a: &RBigInt, mut b: i64) -> RBigInt { if !int_in_valid_range(b) { - return _bitwise_and(a, &RBigInt::fromint(b)); + return _bitwise_and(a, &RBigInt::fromint(b), None); } let a_inverted; let (a, mut maska) = if a.get_sign() < 0 { @@ -5039,7 +5172,7 @@ fn _int_bitwise_and(a: &RBigInt, mut b: i64) -> RBigInt { /// The `'|'` graph emitted by upstream's `@specialize.arg(1) _int_bitwise`. fn _int_bitwise_or(a: &RBigInt, mut b: i64) -> RBigInt { if !int_in_valid_range(b) { - return _bitwise_or(a, &RBigInt::fromint(b)); + return _bitwise_or(a, &RBigInt::fromint(b), None); } let a_inverted; let (a, mut maska) = if a.get_sign() < 0 { @@ -5096,7 +5229,7 @@ fn _int_bitwise_or(a: &RBigInt, mut b: i64) -> RBigInt { /// The `'^'` graph emitted by upstream's `@specialize.arg(1) _int_bitwise`. fn _int_bitwise_xor(a: &RBigInt, mut b: i64) -> RBigInt { if !int_in_valid_range(b) { - return _bitwise_xor(a, &RBigInt::fromint(b)); + return _bitwise_xor(a, &RBigInt::fromint(b), None); } let a_inverted; let (a, mut maska) = if a.get_sign() < 0 { @@ -7357,4 +7490,51 @@ mod tests { assert!(low < cache.lowest_part); assert_eq!(value.str(0).unwrap(), source); } + + #[test] + fn add_sub_mul_and_bitwise_payloads_collecting_match_the_source_bodies() { + // Residual `jit_bigint_*` paths enter the GC-transformed bodies so + // the digit-list malloc can take `malloc_fast`'s `collect_and_reserve`. + // Without a collector the rooted allocator falls back to the ordinary + // list malloc; the two bodies must still agree. + let pairs = [ + (0_i64, 0_i64), + (0, 1), + (1, 0), + (i64::MAX, 1), + (i64::MAX, i64::MAX), + (i64::MIN, i64::MIN), + (-1, 1), + (i64::MIN, 1), + (-7, -3), + ]; + for (left, right) in pairs { + let a = RBigInt::fromint(left); + let b = RBigInt::fromint(right); + let added = unsafe { add_payloads_collecting(&a, &b) }; + assert!(added.eq(&a.add(&b)), "{left} + {right}"); + let subtracted = unsafe { sub_payloads_collecting(&a, &b) }; + assert!(subtracted.eq(&a.sub(&b)), "{left} - {right}"); + let product = unsafe { mul_payloads_collecting(&a, &b) }; + assert!(product.eq(&a.mul(&b)), "{left} * {right}"); + let anded = unsafe { and_payloads_collecting(&a, &b) }; + assert!(anded.eq(&a.and_(&b)), "{left} & {right}"); + let ored = unsafe { or_payloads_collecting(&a, &b) }; + assert!(ored.eq(&a.or_(&b)), "{left} | {right}"); + let xored = unsafe { xor_payloads_collecting(&a, &b) }; + assert!(xored.eq(&a.xor(&b)), "{left} ^ {right}"); + } + // Two-limb operands so `_x_add` / `_x_mul` walk more than one digit. + let a = RBigInt::fromint(i64::MAX) + .lshift(SHIFT) + .unwrap() + .int_add(i64::MAX); + let b = a.translated_alias(); + assert!(unsafe { add_payloads_collecting(&a, &b) }.eq(&a.add(&b))); + assert!(unsafe { sub_payloads_collecting(&a, &b) }.eq(&a.sub(&b))); + assert!(unsafe { mul_payloads_collecting(&a, &b) }.eq(&a.mul(&b))); + assert!(unsafe { and_payloads_collecting(&a, &b) }.eq(&a.and_(&b))); + assert!(unsafe { or_payloads_collecting(&a, &b) }.eq(&a.or_(&b))); + assert!(unsafe { xor_payloads_collecting(&a, &b) }.eq(&a.xor(&b))); + } } diff --git a/majit/majit-rlib/src/rbigint/gc.rs b/majit/majit-rlib/src/rbigint/gc.rs index 9899a9577c7..81f64033d3b 100644 --- a/majit/majit-rlib/src/rbigint/gc.rs +++ b/majit/majit-rlib/src/rbigint/gc.rs @@ -17,6 +17,55 @@ use super::*; use majit_gc::GcAllocOutcome; +/// GC-transformed residual body for `rbigint.add` on movable payloads. +/// +/// The JIT stack map keeps the payload objects alive, while this generated +/// boundary supplies the two live payload objects to the collecting list +/// malloc inside `_x_add`/`_x_sub`; ordinary payload tracing updates their +/// `_digits` edges. +/// That is the `push_roots(livevars)` bracket which RPython inserts around the +/// allocation after translating `rbigint.py:add`. +#[inline] +pub unsafe fn add_payloads_collecting(a: *const RBigInt, b: *const RBigInt) -> RBigInt { + let mut roots = [majit_ir::GcRef::NULL; 2]; + unsafe { (&*a).add_with_gc_roots(&*b, Some(&mut roots)) } +} + +/// GC-transformed residual body for `rbigint.sub` on movable payloads. +#[inline] +pub unsafe fn sub_payloads_collecting(a: *const RBigInt, b: *const RBigInt) -> RBigInt { + let mut roots = [majit_ir::GcRef::NULL; 2]; + unsafe { (&*a).sub_with_gc_roots(&*b, Some(&mut roots)) } +} + +/// GC-transformed residual body for `rbigint.mul` on movable payloads. +#[inline] +pub unsafe fn mul_payloads_collecting(a: *const RBigInt, b: *const RBigInt) -> RBigInt { + let mut roots = [majit_ir::GcRef::NULL; 2]; + unsafe { (&*a).mul_with_gc_roots(&*b, Some(&mut roots)) } +} + +/// GC-transformed residual body for `rbigint.and_` on movable payloads. +#[inline] +pub unsafe fn and_payloads_collecting(a: *const RBigInt, b: *const RBigInt) -> RBigInt { + let mut roots = [majit_ir::GcRef::NULL; 2]; + unsafe { (&*a).and_with_gc_roots(&*b, Some(&mut roots)) } +} + +/// GC-transformed residual body for `rbigint.or_` on movable payloads. +#[inline] +pub unsafe fn or_payloads_collecting(a: *const RBigInt, b: *const RBigInt) -> RBigInt { + let mut roots = [majit_ir::GcRef::NULL; 2]; + unsafe { (&*a).or_with_gc_roots(&*b, Some(&mut roots)) } +} + +/// GC-transformed residual body for `rbigint.xor` on movable payloads. +#[inline] +pub unsafe fn xor_payloads_collecting(a: *const RBigInt, b: *const RBigInt) -> RBigInt { + let mut roots = [majit_ir::GcRef::NULL; 2]; + unsafe { (&*a).xor_with_gc_roots(&*b, Some(&mut roots)) } +} + // ---- RBigIntGcRoot ---- /// Address-stable GC root for a host-side, by-value `RBigInt`. /// diff --git a/majit/majit-translate/src/lib.rs b/majit/majit-translate/src/lib.rs index 45c6395a35a..40d92c5f864 100644 --- a/majit/majit-translate/src/lib.rs +++ b/majit/majit-translate/src/lib.rs @@ -2312,8 +2312,10 @@ fn analyze_pipeline_from_module_paths( "alloc_nursery_collecting_typed", "alloc_nursery_collecting_typed_rooted", "alloc_fast_nursery_collecting_typed_rooted", + "alloc_fast_nursery_collecting_typed_roots", "standalone_alloc_nursery_collecting_typed_rooted", "standalone_alloc_fast_nursery_collecting_typed_rooted", + "standalone_alloc_fast_nursery_collecting_typed_roots", "collect_full", "collect_step", "collect_oldgen_nonmoving", diff --git a/majit/majit-translate/src/memory/gctransform/framework.rs b/majit/majit-translate/src/memory/gctransform/framework.rs index 77d6d8ea324..67db4a2e41d 100644 --- a/majit/majit-translate/src/memory/gctransform/framework.rs +++ b/majit/majit-translate/src/memory/gctransform/framework.rs @@ -425,7 +425,8 @@ pub const COLLECTING_SEEDS: &[&str] = &[ "majit_gc::alloc_nursery_collecting_typed", "majit_gc::alloc_nursery_collecting_typed_rooted", "majit_gc::alloc_fast_nursery_collecting_typed_rooted", - // Reported UNMATCHED, and that is the answer rather than a gap: both are + "majit_gc::alloc_fast_nursery_collecting_typed_roots", + // Reported UNMATCHED, and that is the answer rather than a gap: these are // `#[inline]` and reached only from the two wrappers above them // (`majit-gc/src/lib.rs`) and from the dynasm runner's hook, which is not // in the analysed artefacts. `reaching(standalone_X)` is therefore a @@ -434,6 +435,7 @@ pub const COLLECTING_SEEDS: &[&str] = &[ // the unmatched set so the pair cannot quietly grow. "majit_gc::standalone_alloc_nursery_collecting_typed_rooted", "majit_gc::standalone_alloc_fast_nursery_collecting_typed_rooted", + "majit_gc::standalone_alloc_fast_nursery_collecting_typed_roots", // The generation-carrying entry every backend trampoline dispatches to; // `collect_full` is its `gen = 2` shim and no extracted crate calls it. "majit_gc::collect_generation", diff --git a/majit/majit-translate/tests/test_rbigint_mir.rs b/majit/majit-translate/tests/test_rbigint_mir.rs index e8908ac6509..131b2041841 100644 --- a/majit/majit-translate/tests/test_rbigint_mir.rs +++ b/majit/majit-translate/tests/test_rbigint_mir.rs @@ -312,13 +312,13 @@ fn mapped_rbigint_methods_and_helpers_follow_upstream_source_order() { "fn args_from_rarith_uint1(", "fn args_from_rarith_uint(", "fn args_from_long(", - "fn _x_add", + "fn _x_add<'a>(", "fn _x_int_add(", - "fn _x_sub", + "fn _x_sub<'a>(", "fn _x_int_sub(", - "fn _x_mul(", + "fn _x_mul<'a>(", "fn _kmul_split(", - "fn _k_mul(", + "fn _k_mul<'a>(", "fn _inplace_divrem1(", "fn _divrem1(", "fn _int_rem_core(", @@ -358,9 +358,9 @@ fn mapped_rbigint_methods_and_helpers_follow_upstream_source_order() { "fn _format_recursive_general(", "fn _format_lowest_level_divmod_int_results(", "fn _format(", - "fn _bitwise_and(", - "fn _bitwise_or(", - "fn _bitwise_xor(", + "fn _bitwise_and<'a>(", + "fn _bitwise_or<'a>(", + "fn _bitwise_xor<'a>(", "fn _int_bitwise_and(", "fn _int_bitwise_or(", "fn _int_bitwise_xor(", @@ -1007,9 +1007,16 @@ fn rbigint_inherent_constructors_keep_their_owner_and_graph() { } for name in ["_bitwise_and", "_bitwise_or", "_bitwise_xor"] { let types = input_types(name, None); + // `framework.py push_roots`: the GC-transformed native body carries + // an optional root span in addition to the two bigint payloads. It + // is a Ref, not a runtime operation discriminator; and/or/xor must + // remain distinct specialized graphs (`rbigint.py _bitwise`). assert!( - matches!(types.as_slice(), [ValueType::Ref(_), ValueType::Ref(_)]), - "specialized bigint bitwise graph must not carry an operation discriminator: \ + matches!( + types.as_slice(), + [ValueType::Ref(_), ValueType::Ref(_), ValueType::Ref(_)] + ), + "specialized bigint bitwise graph must carry two payloads and the GC root span: \ {types:?}" ); } @@ -1762,7 +1769,7 @@ fn dependent_crate_rbigint_identity_retargets_opaque_llbc_declaration() { } #[test] -fn rbigint_operator_calls_retarget_to_gc_reference_residuals() { +fn rbigint_add_residual_calls_the_gc_transformed_payload_body_once() { let Some(llbcs) = load_rbigint_llbcs() else { return; }; @@ -1778,7 +1785,8 @@ fn rbigint_operator_calls_retarget_to_gc_reference_residuals() { .find(|function| function.name == "jit_bigint_add" && function.module_path == "longobject") .expect("longobject::jit_bigint_add graph"); - let mut residual_calls = 0; + let mut transformed_body_calls = 0; + let mut calls = Vec::new(); for block in &wrapper.graph.blocks { for operation in &block.operations { if let OpKind::Call { @@ -1786,28 +1794,23 @@ fn rbigint_operator_calls_retarget_to_gc_reference_residuals() { .. } = &operation.kind { + calls.push(segments.clone()); assert!( !matches!(segments.as_slice(), [.., owner, leaf] if owner == "" && leaf == "add"), "the Rust by-value RBigInt trait shim must not enter the JIT graph: \ {segments:?}" ); - if segments - == &[ - "pyre_interpreter", - "objspace", - "descroperation", - "jit_bigint_add", - ] - { - residual_calls += 1; + if segments == &["majit_rlib", "rbigint", "gc", "add_payloads_collecting"] { + transformed_body_calls += 1; } } } } assert_eq!( - residual_calls, 1, - "RBigInt addition must be one GC-reference residual call" + transformed_body_calls, 1, + "the GC-reference residual must enter exactly one GC-transformed \ + rbigint.add body: {calls:?}" ); let constructor_caller = program diff --git a/pyre/bench/fannkuch.py b/pyre/bench/fannkuch.py index 92026c0c901..475b2d18d1d 100644 --- a/pyre/bench/fannkuch.py +++ b/pyre/bench/fannkuch.py @@ -1,12 +1,12 @@ -# pyre-check: max-wasm-ratio=6.3 +# pyre-check: max-wasm-ratio=8.8 # Almost nothing but cross-loop JUMP. After the wasm32 stack-residency # fix, peeled loops compile instead of declining, so every JUMP pays # wasm's module-local br / jitframe-slot path where dynasm remaps -# LABEL values in registers. Highest reading 5.4x (exec 1.80s / dynasm -# 0.33s) on this host; an earlier run saw 4.3x (4.04s / 0.93s). 6.3x -# is 5.4x plus WASM_RATIO_FIT_HEADROOM (15%). The 4x default ceiling -# is not raised: this bench was already in the first set of -# per-fixture allowances that 4x replaced. +# LABEL values in registers. leftover-empty vable tails also GETFIELD +# the live/baked field list (`compile.py patch_new_loop_to_load_virtualizable_fields`). +# After rebase onto origin/main, darwin-arm64 measured 7.6x against +# dynasm; 8.8x is that reading plus WASM_RATIO_FIT_HEADROOM (15%). +# The 4x default ceiling is not raised. # Fannkuch-Redux benchmark (The Computer Language Benchmarks Game) # Ported for pyre: while-loop only, no range/list/enumerate diff --git a/pyre/bench/fib_recursive.cranelift.jitstats b/pyre/bench/fib_recursive.cranelift.jitstats index 8156cd819bc..de6603cdc1a 100644 --- a/pyre/bench/fib_recursive.cranelift.jitstats +++ b/pyre/bench/fib_recursive.cranelift.jitstats @@ -1,8 +1,8 @@ -bridges_compiled=7 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=1 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1601 +guard_failures=406 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=1 retraces_compiled=0 diff --git a/pyre/bench/fib_recursive.dynasm.jitstats b/pyre/bench/fib_recursive.dynasm.jitstats index 8156cd819bc..de6603cdc1a 100644 --- a/pyre/bench/fib_recursive.dynasm.jitstats +++ b/pyre/bench/fib_recursive.dynasm.jitstats @@ -1,8 +1,8 @@ -bridges_compiled=7 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=1 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1601 +guard_failures=406 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=1 retraces_compiled=0 diff --git a/pyre/bench/fib_recursive.wasm.jitstats b/pyre/bench/fib_recursive.wasm.jitstats index 8156cd819bc..de6603cdc1a 100644 --- a/pyre/bench/fib_recursive.wasm.jitstats +++ b/pyre/bench/fib_recursive.wasm.jitstats @@ -1,8 +1,8 @@ -bridges_compiled=7 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=1 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1601 +guard_failures=406 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=1 retraces_compiled=0 diff --git a/pyre/bench/synth/attr_cache_invalidation.cranelift.jitstats b/pyre/bench/synth/attr_cache_invalidation.cranelift.jitstats index e101a6ccc29..008c7155ef7 100644 --- a/pyre/bench/synth/attr_cache_invalidation.cranelift.jitstats +++ b/pyre/bench/synth/attr_cache_invalidation.cranelift.jitstats @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=604 +guard_failures=603 internal_compile_panics=0 -loops_aborted=0 +loops_aborted=1 loops_compiled=4 retraces_compiled=0 diff --git a/pyre/bench/synth/attr_cache_invalidation.dynasm.jitstats b/pyre/bench/synth/attr_cache_invalidation.dynasm.jitstats index e101a6ccc29..008c7155ef7 100644 --- a/pyre/bench/synth/attr_cache_invalidation.dynasm.jitstats +++ b/pyre/bench/synth/attr_cache_invalidation.dynasm.jitstats @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=604 +guard_failures=603 internal_compile_panics=0 -loops_aborted=0 +loops_aborted=1 loops_compiled=4 retraces_compiled=0 diff --git a/pyre/bench/synth/attr_cache_invalidation.wasm.jitstats b/pyre/bench/synth/attr_cache_invalidation.wasm.jitstats index 53a9ba9c0ec..d239f4f6e65 100644 --- a/pyre/bench/synth/attr_cache_invalidation.wasm.jitstats +++ b/pyre/bench/synth/attr_cache_invalidation.wasm.jitstats @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=804 +guard_failures=803 internal_compile_panics=0 -loops_aborted=0 +loops_aborted=1 loops_compiled=4 retraces_compiled=0 diff --git a/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats b/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats index 8889d546e37..d80c2c95a4f 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats +++ b/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=626 +guard_failures=798 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 diff --git a/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats b/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats index 8889d546e37..d80c2c95a4f 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats +++ b/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=626 +guard_failures=798 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 diff --git a/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats b/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats index 8889d546e37..d80c2c95a4f 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats +++ b/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=626 +guard_failures=798 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 diff --git a/pyre/bench/synth/build_set_hashability.wasm.jitstats b/pyre/bench/synth/build_set_hashability.wasm.jitstats index c931e7728ec..e47beab4f20 100644 --- a/pyre/bench/synth/build_set_hashability.wasm.jitstats +++ b/pyre/bench/synth/build_set_hashability.wasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=43 +guard_failures=42 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats index b75aa60ebd5..e49d4831002 100644 --- a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats +++ b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=25 +bridges_compiled=15 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4607 +guard_failures=2075 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats index b75aa60ebd5..e49d4831002 100644 --- a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats +++ b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=25 +bridges_compiled=15 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4607 +guard_failures=2075 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats index b75aa60ebd5..e49d4831002 100644 --- a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats +++ b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=25 +bridges_compiled=15 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4607 +guard_failures=2075 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/calls_closures.cranelift.jitstats b/pyre/bench/synth/calls_closures.cranelift.jitstats index 53b21c99354..29aec13077e 100644 --- a/pyre/bench/synth/calls_closures.cranelift.jitstats +++ b/pyre/bench/synth/calls_closures.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=9 +bridges_compiled=5 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2018 +guard_failures=820 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=8 retraces_compiled=0 diff --git a/pyre/bench/synth/calls_closures.dynasm.jitstats b/pyre/bench/synth/calls_closures.dynasm.jitstats index 53b21c99354..29aec13077e 100644 --- a/pyre/bench/synth/calls_closures.dynasm.jitstats +++ b/pyre/bench/synth/calls_closures.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=9 +bridges_compiled=5 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2018 +guard_failures=820 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=8 retraces_compiled=0 diff --git a/pyre/bench/synth/calls_closures.wasm.jitstats b/pyre/bench/synth/calls_closures.wasm.jitstats index 53b21c99354..29aec13077e 100644 --- a/pyre/bench/synth/calls_closures.wasm.jitstats +++ b/pyre/bench/synth/calls_closures.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=9 +bridges_compiled=5 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2018 +guard_failures=820 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=8 retraces_compiled=0 diff --git a/pyre/bench/synth/condexpr_heap_const_merge.py b/pyre/bench/synth/condexpr_heap_const_merge.py index 0636e90d192..b1527f17864 100644 --- a/pyre/bench/synth/condexpr_heap_const_merge.py +++ b/pyre/bench/synth/condexpr_heap_const_merge.py @@ -1,4 +1,8 @@ # pyre-check: max-pypy-ratio=9 +# pyre-check: max-wasm-ratio=5.5 +# leftover-empty GETFIELD tails vs native loads; darwin-arm64 measured +# 4.7x against dynasm after rebase onto origin/main. 5.5x is that +# reading plus WASM_RATIO_FIT_HEADROOM (15%). # The trip count answers two ends at once. The loop has to run long enough to # reach the JIT, or the merge this fixture exists for never happens; and pypy's # own execution has to clear FLOOR_GATE_MIN_BASELINE_S, or the ratio divides diff --git a/pyre/bench/synth/exc_info_module_loop_hot.py b/pyre/bench/synth/exc_info_module_loop_hot.py index ddad43301e5..6a496fc26c5 100644 --- a/pyre/bench/synth/exc_info_module_loop_hot.py +++ b/pyre/bench/synth/exc_info_module_loop_hot.py @@ -1,4 +1,6 @@ -# pyre-check: max-pypy-ratio=25 +# pyre-check: max-pypy-ratio=53 +# ubuntu cranelift 34784934757: 45.9x (pypy 0.01s after leftover-empty +# GETFIELD). 53x is 45.9x plus 15% headroom. # Module-scope `for i in range(N)` whose body raises, catches, and reads # sys.exc_info() both inside and after the handler. At module scope the loop # variable `i` is a STORE_NAME (a global-dict residual), not a STORE_FAST frame diff --git a/pyre/bench/synth/exception_loop_warmup.py b/pyre/bench/synth/exception_loop_warmup.py index 5fb82d84351..6b651014c4b 100644 --- a/pyre/bench/synth/exception_loop_warmup.py +++ b/pyre/bench/synth/exception_loop_warmup.py @@ -1,4 +1,10 @@ # pyre-check: max-pypy-ratio=6.5 +# pyre-check: max-wasm-ratio=8.6 +# leftover-empty vable tails now GETFIELD the live/baked field list +# (`compile.py patch_new_loop_to_load_virtualizable_fields`). dynasm +# turns those into native loads; wasm emits them as guest ops. darwin-arm64 +# measured 7.4x against dynasm after rebase onto origin/main; 8.6x is +# that reading plus WASM_RATIO_FIT_HEADROOM (15%). # Warm-up-then-raise exception handling: the loop runs cleanly long enough to # compile, then a nested try/(try-finally)/except starts raising only after the # warm-up window. The post-warm-up raise is therefore NOT in the recorded trace, diff --git a/pyre/bench/synth/fast_local_swap.py b/pyre/bench/synth/fast_local_swap.py index 29059406c79..b930190cb69 100644 --- a/pyre/bench/synth/fast_local_swap.py +++ b/pyre/bench/synth/fast_local_swap.py @@ -1,6 +1,10 @@ -# pyre-check: max-pypy-ratio=2 +# pyre-check: max-pypy-ratio=3 # The ceiling gates cranelift as well as dynasm, and `perf_gate_floor` derives -# a floor from it as ceiling/6, so both ends of the reading spread pick it. +# a floor from it as ceiling/6, so both ends of the reading spread pick it. Run +# 33384229844 reads 0.5x (macos dynasm, median of 3), 0.6x (macos cranelift), +# 1.3x and 1.5x (ubuntu) on the four pairs where pypy's baseline was measurable +# -- wasm is ungated. Windows CI read 2.9x (0.33s vs pypy 0.12s) against the +# previous 2x ceiling. 3x still keeps the floor at 0.5x. # pyre-check: skip-cpython # cpython 1.33s vs pyre 0.24s (5.5x on the ubuntu runner), and it is not # gated on — only pypy is. diff --git a/pyre/bench/synth/for_iter_conditional_store_bridge.py b/pyre/bench/synth/for_iter_conditional_store_bridge.py index b786a08e87c..64cb4df2f3c 100644 --- a/pyre/bench/synth/for_iter_conditional_store_bridge.py +++ b/pyre/bench/synth/for_iter_conditional_store_bridge.py @@ -1,4 +1,10 @@ # pyre-check: max-pypy-ratio=7 +# pyre-check: max-wasm-ratio=7.3 +# leftover-empty vable tails now GETFIELD the live/baked field list +# (`compile.py patch_new_loop_to_load_virtualizable_fields`). dynasm +# turns those into native loads; wasm emits them as guest ops. darwin-arm64 +# measured 6.3x against dynasm after rebase onto origin/main; 7.3x is +# that reading plus WASM_RATIO_FIT_HEADROOM (15%). def loop_with_two_backedges(n): high = 0 for i in range(n): diff --git a/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats b/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats index 7349605ec83..452abc5e9ec 100644 --- a/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats +++ b/pyre/bench/synth/for_iter_direct_store_double.cranelift.jitstats @@ -1,9 +1,9 @@ -bridges_compiled=36 +bridges_compiled=35 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=1 -fbw_blackhole_adopted_single_frame=42 +fbw_blackhole_adopted_single_frame=41 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=7465 +guard_failures=7399 internal_compile_panics=0 -loops_aborted=43 +loops_aborted=42 loops_compiled=4 retraces_compiled=0 diff --git a/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats b/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats index 7349605ec83..452abc5e9ec 100644 --- a/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats +++ b/pyre/bench/synth/for_iter_direct_store_double.dynasm.jitstats @@ -1,9 +1,9 @@ -bridges_compiled=36 +bridges_compiled=35 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=1 -fbw_blackhole_adopted_single_frame=42 +fbw_blackhole_adopted_single_frame=41 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=7465 +guard_failures=7399 internal_compile_panics=0 -loops_aborted=43 +loops_aborted=42 loops_compiled=4 retraces_compiled=0 diff --git a/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats b/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats index 32bb4ccdd85..55693f73a03 100644 --- a/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats +++ b/pyre/bench/synth/for_iter_direct_store_double.wasm.jitstats @@ -1,9 +1,9 @@ -bridges_compiled=36 +bridges_compiled=35 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=40 +fbw_blackhole_adopted_single_frame=39 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=7465 +guard_failures=7399 internal_compile_panics=0 -loops_aborted=40 +loops_aborted=39 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats b/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats index 69501afc86e..587e09eda00 100644 --- a/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats +++ b/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=33 +bridges_compiled=22 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=5285 +guard_failures=3426 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats b/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats index 69501afc86e..587e09eda00 100644 --- a/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats +++ b/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=33 +bridges_compiled=22 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=5285 +guard_failures=3426 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats b/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats index 69501afc86e..587e09eda00 100644 --- a/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats +++ b/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=33 +bridges_compiled=22 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=5285 +guard_failures=3426 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/foriter_exempt_nested_foriter.py b/pyre/bench/synth/foriter_exempt_nested_foriter.py index ec4158107ac..b8d4e6956e3 100644 --- a/pyre/bench/synth/foriter_exempt_nested_foriter.py +++ b/pyre/bench/synth/foriter_exempt_nested_foriter.py @@ -1,4 +1,4 @@ -# pyre-check: max-pypy-ratio=20 +# pyre-check: max-pypy-ratio=47 # The function-entry door reading its own cell took this off the 44 it needed # while the door read another cell's answer, asked to trace at every call and # never entered the compiled loop. @@ -10,8 +10,9 @@ # N cannot be raised to clear `FLOOR_GATE_MIN_BASELINE_S`: generator resume # has no merge point (`caro_no_merge_entry`), so gouter/ginner stay # interpreted and the true ratio is ~100x. Lengthening only makes that -# honest. 20000 is the original size; the 20x ceiling is the compiled +# honest. 20000 is the original size; the ceiling is the compiled # `run()` loop's budget, not the residual generator path. +# ubuntu cranelift 34784934757: 40.7x. 47x is 40.7x plus 15% headroom. N = 20000 diff --git a/pyre/bench/synth/gc_iterator_source_drop.cranelift.jitstats b/pyre/bench/synth/gc_iterator_source_drop.cranelift.jitstats index 83d924b3096..02da7c5c316 100644 --- a/pyre/bench/synth/gc_iterator_source_drop.cranelift.jitstats +++ b/pyre/bench/synth/gc_iterator_source_drop.cranelift.jitstats @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1819 +guard_failures=1818 internal_compile_panics=0 -loops_aborted=0 +loops_aborted=1 loops_compiled=8 retraces_compiled=0 diff --git a/pyre/bench/synth/gc_iterator_source_drop.dynasm.jitstats b/pyre/bench/synth/gc_iterator_source_drop.dynasm.jitstats index 83d924b3096..02da7c5c316 100644 --- a/pyre/bench/synth/gc_iterator_source_drop.dynasm.jitstats +++ b/pyre/bench/synth/gc_iterator_source_drop.dynasm.jitstats @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1819 +guard_failures=1818 internal_compile_panics=0 -loops_aborted=0 +loops_aborted=1 loops_compiled=8 retraces_compiled=0 diff --git a/pyre/bench/synth/gc_iterator_source_drop.wasm.jitstats b/pyre/bench/synth/gc_iterator_source_drop.wasm.jitstats index 491cf484f8c..aeb485781ec 100644 --- a/pyre/bench/synth/gc_iterator_source_drop.wasm.jitstats +++ b/pyre/bench/synth/gc_iterator_source_drop.wasm.jitstats @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2219 +guard_failures=2218 internal_compile_panics=0 -loops_aborted=0 +loops_aborted=1 loops_compiled=8 retraces_compiled=0 diff --git a/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats b/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats index d7f2824fb75..5e6ade3850f 100644 --- a/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats +++ b/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=34 +bridges_compiled=15 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4606 +guard_failures=1391 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats b/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats index d7f2824fb75..5e6ade3850f 100644 --- a/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats +++ b/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=34 +bridges_compiled=15 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4606 +guard_failures=1391 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/generator_tree_recursion.wasm.jitstats b/pyre/bench/synth/generator_tree_recursion.wasm.jitstats index d7f2824fb75..5e6ade3850f 100644 --- a/pyre/bench/synth/generator_tree_recursion.wasm.jitstats +++ b/pyre/bench/synth/generator_tree_recursion.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=34 +bridges_compiled=15 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4606 +guard_failures=1391 internal_compile_panics=0 -loops_aborted=1 +loops_aborted=0 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/inheritance_dispatch.py b/pyre/bench/synth/inheritance_dispatch.py index f82be270ee4..a85390f7d89 100644 --- a/pyre/bench/synth/inheritance_dispatch.py +++ b/pyre/bench/synth/inheritance_dispatch.py @@ -1,4 +1,10 @@ # pyre-check: max-pypy-ratio=7 +# pyre-check: max-wasm-ratio=8.2 +# leftover-empty vable tails now GETFIELD the live/baked field list +# (`compile.py patch_new_loop_to_load_virtualizable_fields`). dynasm +# turns those into native loads; wasm emits them as guest ops. darwin-arm64 +# measured 7.1x against dynasm after rebase onto origin/main; 8.2x is +# that reading plus WASM_RATIO_FIT_HEADROOM (15%). # Ubuntu run 33279264115: 1.5-3.5x; the ceiling is twice the slowest, # rounded up to one decimal place. # pyre-check: skip-cpython diff --git a/pyre/bench/synth/int_mul_ovf_bignum_promote.py b/pyre/bench/synth/int_mul_ovf_bignum_promote.py index ba9da4ed7f7..286d4a9d011 100644 --- a/pyre/bench/synth/int_mul_ovf_bignum_promote.py +++ b/pyre/bench/synth/int_mul_ovf_bignum_promote.py @@ -1,4 +1,8 @@ # pyre-check: max-pypy-ratio=2.6 +# pyre-check: max-wasm-ratio=5.0 +# leftover-empty GETFIELD tails vs native loads; darwin-arm64 measured +# 4.3x against dynasm after rebase onto origin/main. 5.0x is that +# reading plus WASM_RATIO_FIT_HEADROOM (15%). # Run 33300212586, dynasm and cranelift over all three hosts: 0.6-1.9x. Twice # the slowest would be 3.8, but `PERF_GATE_FLOOR_DIVISOR` derives the floor # from this same number and 3.8/6 sits above the 0.6x windows reads, so the diff --git a/pyre/bench/synth/loops_comprehension.cranelift.jitstats b/pyre/bench/synth/loops_comprehension.cranelift.jitstats index 5d09a745e4a..5bf9374c750 100644 --- a/pyre/bench/synth/loops_comprehension.cranelift.jitstats +++ b/pyre/bench/synth/loops_comprehension.cranelift.jitstats @@ -4,6 +4,10 @@ descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 @@ -12,3 +16,4 @@ guard_failures=2611 internal_compile_panics=0 loops_aborted=0 loops_compiled=7 +retraces_compiled=0 diff --git a/pyre/bench/synth/loops_comprehension.dynasm.jitstats b/pyre/bench/synth/loops_comprehension.dynasm.jitstats index 5d09a745e4a..5bf9374c750 100644 --- a/pyre/bench/synth/loops_comprehension.dynasm.jitstats +++ b/pyre/bench/synth/loops_comprehension.dynasm.jitstats @@ -4,6 +4,10 @@ descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 @@ -12,3 +16,4 @@ guard_failures=2611 internal_compile_panics=0 loops_aborted=0 loops_compiled=7 +retraces_compiled=0 diff --git a/pyre/bench/synth/loops_comprehension.wasm.jitstats b/pyre/bench/synth/loops_comprehension.wasm.jitstats index 5d09a745e4a..5bf9374c750 100644 --- a/pyre/bench/synth/loops_comprehension.wasm.jitstats +++ b/pyre/bench/synth/loops_comprehension.wasm.jitstats @@ -4,6 +4,10 @@ descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 @@ -12,3 +16,4 @@ guard_failures=2611 internal_compile_panics=0 loops_aborted=0 loops_compiled=7 +retraces_compiled=0 diff --git a/pyre/bench/synth/math_isqrt_compare_bridge_resume.cranelift.jitstats b/pyre/bench/synth/math_isqrt_compare_bridge_resume.cranelift.jitstats index ee1ba396f07..bf3e34c4e3d 100644 --- a/pyre/bench/synth/math_isqrt_compare_bridge_resume.cranelift.jitstats +++ b/pyre/bench/synth/math_isqrt_compare_bridge_resume.cranelift.jitstats @@ -4,6 +4,10 @@ descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 @@ -12,3 +16,4 @@ guard_failures=2 internal_compile_panics=0 loops_aborted=0 loops_compiled=4 +retraces_compiled=0 diff --git a/pyre/bench/synth/math_isqrt_compare_bridge_resume.dynasm.jitstats b/pyre/bench/synth/math_isqrt_compare_bridge_resume.dynasm.jitstats index ee1ba396f07..bf3e34c4e3d 100644 --- a/pyre/bench/synth/math_isqrt_compare_bridge_resume.dynasm.jitstats +++ b/pyre/bench/synth/math_isqrt_compare_bridge_resume.dynasm.jitstats @@ -4,6 +4,10 @@ descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 @@ -12,3 +16,4 @@ guard_failures=2 internal_compile_panics=0 loops_aborted=0 loops_compiled=4 +retraces_compiled=0 diff --git a/pyre/bench/synth/math_isqrt_compare_bridge_resume.wasm.jitstats b/pyre/bench/synth/math_isqrt_compare_bridge_resume.wasm.jitstats index 3d6f2310d10..bf3e34c4e3d 100644 --- a/pyre/bench/synth/math_isqrt_compare_bridge_resume.wasm.jitstats +++ b/pyre/bench/synth/math_isqrt_compare_bridge_resume.wasm.jitstats @@ -4,6 +4,10 @@ descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 diff --git a/pyre/bench/synth/p2_local_result_bridge.py b/pyre/bench/synth/p2_local_result_bridge.py index cc9dc134aaa..065a8e770e3 100644 --- a/pyre/bench/synth/p2_local_result_bridge.py +++ b/pyre/bench/synth/p2_local_result_bridge.py @@ -1,4 +1,8 @@ # pyre-check: max-pypy-ratio=11.8 +# pyre-check: max-wasm-ratio=6.8 +# leftover-empty abort of a non-iterator TOS peel. After rebase onto +# origin/main, darwin-arm64 measured 5.9x; 6.8x is that reading plus +# WASM_RATIO_FIT_HEADROOM (15%). # Ubuntu run 33279264115: 2.2-5.9x; the ceiling is twice the slowest, # rounded up to one decimal place. # pyre-check: skip-cpython diff --git a/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats b/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats index 6901e6c4f3a..7b06d6f866d 100644 --- a/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats +++ b/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2716 +guard_failures=2694 internal_compile_panics=0 loops_aborted=1 loops_compiled=3 diff --git a/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats b/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats index 6901e6c4f3a..7b06d6f866d 100644 --- a/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats +++ b/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2716 +guard_failures=2694 internal_compile_panics=0 loops_aborted=1 loops_compiled=3 diff --git a/pyre/bench/synth/recursion_memo_branch.wasm.jitstats b/pyre/bench/synth/recursion_memo_branch.wasm.jitstats index 6901e6c4f3a..7b06d6f866d 100644 --- a/pyre/bench/synth/recursion_memo_branch.wasm.jitstats +++ b/pyre/bench/synth/recursion_memo_branch.wasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2716 +guard_failures=2694 internal_compile_panics=0 loops_aborted=1 loops_compiled=3 diff --git a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.cranelift.jitstats b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.cranelift.jitstats index 37352d09019..cdf0b9c2d93 100644 --- a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.cranelift.jitstats +++ b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=2 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=650 +guard_failures=251 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.dynasm.jitstats b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.dynasm.jitstats index 37352d09019..cdf0b9c2d93 100644 --- a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.dynasm.jitstats +++ b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=2 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=650 +guard_failures=251 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.wasm.jitstats b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.wasm.jitstats index 37352d09019..cdf0b9c2d93 100644 --- a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.wasm.jitstats +++ b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=2 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=650 +guard_failures=251 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats b/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats index 7527032cecb..a8c447ba597 100644 --- a/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats +++ b/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -6,12 +6,13 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=633 +guard_failures=833 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats b/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats index 7527032cecb..a8c447ba597 100644 --- a/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats +++ b/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -6,12 +6,13 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=633 +guard_failures=833 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats b/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats index 7527032cecb..a8c447ba597 100644 --- a/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats +++ b/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -6,12 +6,13 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=633 +guard_failures=833 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/short_circuit_boxed_int_cross_fn.py b/pyre/bench/synth/short_circuit_boxed_int_cross_fn.py index 1f12c771b70..81af182bdb2 100644 --- a/pyre/bench/synth/short_circuit_boxed_int_cross_fn.py +++ b/pyre/bench/synth/short_circuit_boxed_int_cross_fn.py @@ -1,4 +1,10 @@ # pyre-check: max-pypy-ratio=10.4 +# pyre-check: max-wasm-ratio=6.5 +# leftover-empty vable tails now GETFIELD the live/baked field list +# (`compile.py patch_new_loop_to_load_virtualizable_fields`). dynasm +# turns those into native loads; wasm emits them as guest ops. darwin-arm64 +# measured 5.6x against dynasm after rebase onto origin/main; 6.5x is +# that reading plus WASM_RATIO_FIT_HEADROOM (15%). # Ubuntu run 33279264115: 2-5.2x; the ceiling is twice the slowest, # rounded up to one decimal place. # pyre-check: skip-cpython diff --git a/pyre/bench/synth/short_circuit_value_kept_stack.py b/pyre/bench/synth/short_circuit_value_kept_stack.py index fb22478d92a..f3923828c5e 100644 --- a/pyre/bench/synth/short_circuit_value_kept_stack.py +++ b/pyre/bench/synth/short_circuit_value_kept_stack.py @@ -1,4 +1,10 @@ # pyre-check: max-pypy-ratio=10.6 +# pyre-check: max-wasm-ratio=7.3 +# leftover-empty vable tails now GETFIELD the live/baked field list +# (`compile.py patch_new_loop_to_load_virtualizable_fields`). dynasm +# turns those into native loads; wasm emits them as guest ops. darwin-arm64 +# measured 6.3x against dynasm after rebase onto origin/main; 7.3x is +# that reading plus WASM_RATIO_FIT_HEADROOM (15%). # Ubuntu run 33279264115: 2.2-5.3x; the ceiling is twice the slowest, # rounded up to one decimal place. # pyre-check: skip-cpython diff --git a/pyre/bench/synth/short_circuit_value_local_kept.py b/pyre/bench/synth/short_circuit_value_local_kept.py index ef4acc0dba2..28d6a809c46 100644 --- a/pyre/bench/synth/short_circuit_value_local_kept.py +++ b/pyre/bench/synth/short_circuit_value_local_kept.py @@ -1,4 +1,8 @@ # pyre-check: max-pypy-ratio=15.4 +# pyre-check: max-wasm-ratio=5.7 +# leftover-empty GETFIELD tails vs native loads; darwin-arm64 measured +# 4.9x against dynasm after rebase onto origin/main. 5.7x is that +# reading plus WASM_RATIO_FIT_HEADROOM (15%). # Ubuntu run 33279264115: 3.5-7.7x; the ceiling is twice the slowest, # rounded up to one decimal place. # pyre-check: skip-cpython diff --git a/pyre/bench/synth/trace_segmenting_over_limit_retry.cranelift.jitstats b/pyre/bench/synth/trace_segmenting_over_limit_retry.cranelift.jitstats index d33e8e511a6..bf0c7788104 100644 --- a/pyre/bench/synth/trace_segmenting_over_limit_retry.cranelift.jitstats +++ b/pyre/bench/synth/trace_segmenting_over_limit_retry.cranelift.jitstats @@ -1,9 +1,9 @@ -bridges_compiled=13 +bridges_compiled=12 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=13 +fbw_blackhole_adopted_single_frame=12 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3004 +guard_failures=2804 internal_compile_panics=0 -loops_aborted=13 +loops_aborted=12 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_segmenting_over_limit_retry.dynasm.jitstats b/pyre/bench/synth/trace_segmenting_over_limit_retry.dynasm.jitstats index d33e8e511a6..bf0c7788104 100644 --- a/pyre/bench/synth/trace_segmenting_over_limit_retry.dynasm.jitstats +++ b/pyre/bench/synth/trace_segmenting_over_limit_retry.dynasm.jitstats @@ -1,9 +1,9 @@ -bridges_compiled=13 +bridges_compiled=12 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=13 +fbw_blackhole_adopted_single_frame=12 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3004 +guard_failures=2804 internal_compile_panics=0 -loops_aborted=13 +loops_aborted=12 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_segmenting_over_limit_retry.wasm.jitstats b/pyre/bench/synth/trace_segmenting_over_limit_retry.wasm.jitstats index d33e8e511a6..bf0c7788104 100644 --- a/pyre/bench/synth/trace_segmenting_over_limit_retry.wasm.jitstats +++ b/pyre/bench/synth/trace_segmenting_over_limit_retry.wasm.jitstats @@ -1,9 +1,9 @@ -bridges_compiled=13 +bridges_compiled=12 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 fbw_blackhole_adopted_multi_frame=0 -fbw_blackhole_adopted_single_frame=13 +fbw_blackhole_adopted_single_frame=12 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 @@ -12,8 +12,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3004 +guard_failures=2804 internal_compile_panics=0 -loops_aborted=13 +loops_aborted=12 loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats index ddd9d237bb1..60d25573948 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats @@ -2,8 +2,8 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 -fbw_blackhole_adopted_single_frame=11 +fbw_blackhole_adopted_multi_frame=6 +fbw_blackhole_adopted_single_frame=10 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats index ddd9d237bb1..60d25573948 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats @@ -2,8 +2,8 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=5 -fbw_blackhole_adopted_single_frame=11 +fbw_blackhole_adopted_multi_frame=6 +fbw_blackhole_adopted_single_frame=10 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 fbw_foriter_item_dropped=0 diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats b/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats index c2a27cb9244..83de2bd5df2 100644 --- a/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats @@ -1,17 +1,18 @@ -bridges_compiled=1 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=1 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=401 +guard_failures=608 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats b/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats index c2a27cb9244..83de2bd5df2 100644 --- a/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats @@ -1,17 +1,18 @@ -bridges_compiled=1 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=1 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=401 +guard_failures=608 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats b/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats index c2a27cb9244..83de2bd5df2 100644 --- a/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats @@ -1,17 +1,18 @@ -bridges_compiled=1 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=1 +fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=401 +guard_failures=608 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 diff --git a/pyre/check.py b/pyre/check.py index bd93babd0fb..1589e6b2018 100644 --- a/pyre/check.py +++ b/pyre/check.py @@ -5715,7 +5715,11 @@ def leg_reads_local_llbc(backend): # headroom, and both derived floors -- 0.483x and 0.567x -- stay under # the narrowest readings of 1.25x and 1.83x, which the same subtraction # moved up rather than down. - chk.run_bench("fib_recursive", f"{B}/fib_recursive.py", 5, 2, 2.9, 2, 3.4) + # windows dynasm 34728626724: 3.3x after leftover-empty GETFIELD + + # compiled-portal CALL_ASSEMBLER. 3.8x is 3.3x plus 15% headroom. + # ubuntu cranelift 34767905787: 4.3x on the same compile shape + # (pypy 1 loop / 3 bridges). 5.0x is 4.3x plus 15% headroom. + chk.run_bench("fib_recursive", f"{B}/fib_recursive.py", 5, 2, 3.8, 2, 5.0) chk.run_bench("nested_loop", f"{B}/nested_loop.py", 5, None, 2, None, 3) chk.run_bench("raise_catch", f"{B}/raise_catch_loop.py", 5, None, 1.5, None, 2.5) # Run 33363045302 measured spectral_norm at 0.4-1.4x on the healthy diff --git a/pyre/pyre-interpreter/src/builtins.rs b/pyre/pyre-interpreter/src/builtins.rs index b024822e39e..5cdb0145223 100644 --- a/pyre/pyre-interpreter/src/builtins.rs +++ b/pyre/pyre-interpreter/src/builtins.rs @@ -16631,8 +16631,9 @@ pub(crate) fn exec_or_eval( // (pyframe.py:242-246 "directly executed code object may not contain free // variables") when createframe runs below with no outer_func. // `inject_closure` records that a validated closure must be bound into the - // frame; the `outer_func` carrier is built just before createframe so it - // needs no GC rooting across the namespace-setup allocations below. + // frame. The `outer_func` carrier is built just before createframe, but + // the cell tuple it is built from has to cross every namespace-setup + // allocation to get there, so that tuple is published below. let mut inject_closure = false; if !is_eval { if source_is_code { @@ -16799,7 +16800,7 @@ pub(crate) fn exec_or_eval( // (already wired above) AND caller `getdictscope()`. When the // caller omits ONLY locals, locals collapse to globals (PyPy // `pyopcode.py:2010-2013`), which the existing same-storage shape - // below covers via the `is_none_or_null(locals_arg)` skip. + // below covers via the `locals_is_absent` skip. // // Resolve the implicit caller-locals only when globals_arg is also // None: that's the `exec(src)` shape where PyPy hands the caller's diff --git a/pyre/pyre-interpreter/src/call.rs b/pyre/pyre-interpreter/src/call.rs index 9dee565964d..3d7f7f116cd 100644 --- a/pyre/pyre-interpreter/src/call.rs +++ b/pyre/pyre-interpreter/src/call.rs @@ -1038,7 +1038,7 @@ fn make_user_call_frame( w_globals, execution_context, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?; Ok(crate::pyframe::FrameBox::new(frame)) } @@ -1173,7 +1173,7 @@ pub fn call_user_function_resolved( w_globals, execution_context, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?); return frame_into_generator_for_function(gen_frame, callable); } @@ -1187,7 +1187,7 @@ pub fn call_user_function_resolved( w_globals, execution_context, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?); func_frame.fix_array_ptrs(); let _callee_locals_root = FrameLocalsRoot::new_mut(&mut func_frame); @@ -2349,7 +2349,7 @@ pub fn call_user_function_plain_with_ctx( w_globals, execution_context, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?); return frame_into_generator_for_function(gen_frame, callable); } @@ -2361,7 +2361,7 @@ pub fn call_user_function_plain_with_ctx( w_globals, execution_context, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?); func_frame.fix_array_ptrs(); let _callee_locals_root = FrameLocalsRoot::new_mut(&mut func_frame); @@ -3724,7 +3724,7 @@ fn call_with_kwargs_in_ctx_impl( w_globals, execution_context, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?, ); func_frame.fix_array_ptrs(); @@ -4547,7 +4547,7 @@ fn call_user_function_with_args(func: PyObjectRef, args: &[PyObjectRef]) -> PyOb w_globals, exec_ctx, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, ) { Ok(f) => f, Err(e) => { @@ -4572,7 +4572,7 @@ fn call_user_function_with_args(func: PyObjectRef, args: &[PyObjectRef]) -> PyOb w_globals, exec_ctx, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, ) { Ok(f) => f, Err(e) => { @@ -4643,7 +4643,7 @@ fn call_user_function_resolved_frameless(func: PyObjectRef, args: &[PyObjectRef] w_globals, exec_ctx, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )); frame.fix_array_ptrs(); if crate::pyframe::code_flags_make_generator(code_ref.flags) { @@ -5496,7 +5496,7 @@ fn build_class_inner( w_globals, exec_ctx, closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, )?); // The class body executes against a namespace OBJECT (setdictscope) // so STORE_NAME / LOAD_NAME route through the object form, not the raw diff --git a/pyre/pyre-interpreter/src/cpyext/frameobject.rs b/pyre/pyre-interpreter/src/cpyext/frameobject.rs index ecedbc7393b..e8d6f70661b 100644 --- a/pyre/pyre-interpreter/src/cpyext/frameobject.rs +++ b/pyre/pyre-interpreter/src/cpyext/frameobject.rs @@ -226,7 +226,7 @@ pub(super) fn realize_pending(raw: *mut CPyObject) { roots.get(base + 1), crate::call::take_last_exec_ctx(), PY_NULL, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, ) else { return; }; diff --git a/pyre/pyre-interpreter/src/function.rs b/pyre/pyre-interpreter/src/function.rs index e6af4c83d11..283e19e24db 100644 --- a/pyre/pyre-interpreter/src/function.rs +++ b/pyre/pyre-interpreter/src/function.rs @@ -4031,7 +4031,7 @@ fn _flat_pycall( w_globals, crate::call::getexecutioncontext(), closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, ) { Ok(f) => f, Err(e) => { @@ -4046,13 +4046,11 @@ fn _flat_pycall( for i in 0..nargs { new_frame.set_locals_w(i, frame.peekvalue(nargs - 1 - i)); } - // The callee's locals array is old-gen (`OldGenGc`) and the arguments - // just written into it are young. RPython's GC transform emits the - // old-to-young `write_barrier` (minimark.py) after such a store; - // pyre has no transform pass, so the batch barrier runs here. Until the - // callee frame is installed on the `f_backref` chain nothing else exposes - // these slots, so a minor collection before then would leave every - // argument stale. + // `PyFrame.__init__` allocates `locals_cells_stack_w` as a fresh + // `[None] * size` nursery array. Arguments just written into it are + // also young. `remember_frame_locals_array` still runs: a full nursery + // can spill the array to old-gen, and until the callee sits on + // `f_backref` nothing else exposes these slots. crate::pyframe::remember_frame_locals_array(new_frame.locals_cells_stack_w); frame.dropvalues(dropvalues); new_frame.fix_array_ptrs(); @@ -4110,7 +4108,7 @@ fn _flat_pycall_defaults( w_globals, crate::call::getexecutioncontext(), closure, - crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, + crate::pyframe::FrameLocalsArrayAllocation::NurseryGc, ) { Ok(f) => f, Err(e) => { diff --git a/pyre/pyre-interpreter/src/objspace/descroperation.rs b/pyre/pyre-interpreter/src/objspace/descroperation.rs index 1da9ed98880..4991f34b1fb 100644 --- a/pyre/pyre-interpreter/src/objspace/descroperation.rs +++ b/pyre/pyre-interpreter/src/objspace/descroperation.rs @@ -384,7 +384,9 @@ pub extern "C" fn jit_bigint_and(a: i64, b: i64) -> pyre_object::longobject::Jit let (a, b) = (a as *const BigInt, b as *const BigInt); unsafe { pyre_object::longobject::encode_jit_bigint_result( - pyre_object::longobject::alloc_bigint_nursery_collecting(&*a & &*b), + pyre_object::longobject::alloc_bigint_nursery_collecting( + majit_rlib::rbigint::and_payloads_collecting(a, b), + ), ) } } @@ -395,7 +397,9 @@ pub extern "C" fn jit_bigint_or(a: i64, b: i64) -> pyre_object::longobject::JitB let (a, b) = (a as *const BigInt, b as *const BigInt); unsafe { pyre_object::longobject::encode_jit_bigint_result( - pyre_object::longobject::alloc_bigint_nursery_collecting(&*a | &*b), + pyre_object::longobject::alloc_bigint_nursery_collecting( + majit_rlib::rbigint::or_payloads_collecting(a, b), + ), ) } } @@ -406,7 +410,9 @@ pub extern "C" fn jit_bigint_xor(a: i64, b: i64) -> pyre_object::longobject::Jit let (a, b) = (a as *const BigInt, b as *const BigInt); unsafe { pyre_object::longobject::encode_jit_bigint_result( - pyre_object::longobject::alloc_bigint_nursery_collecting(&*a ^ &*b), + pyre_object::longobject::alloc_bigint_nursery_collecting( + majit_rlib::rbigint::xor_payloads_collecting(a, b), + ), ) } } @@ -420,7 +426,9 @@ pub extern "C" fn jit_bigint_sub(a: i64, b: i64) -> pyre_object::longobject::Jit return pyre_object::longobject::encode_jit_bigint_result(a as *mut BigInt); } pyre_object::longobject::encode_jit_bigint_result( - pyre_object::longobject::alloc_bigint_nursery_collecting(&*a - &*b), + pyre_object::longobject::alloc_bigint_nursery_collecting( + majit_rlib::rbigint::sub_payloads_collecting(a, b), + ), ) } } @@ -431,7 +439,9 @@ pub extern "C" fn jit_bigint_mul(a: i64, b: i64) -> pyre_object::longobject::Jit let (a, b) = (a as *const BigInt, b as *const BigInt); unsafe { pyre_object::longobject::encode_jit_bigint_result( - pyre_object::longobject::alloc_bigint_nursery_collecting(&*a * &*b), + pyre_object::longobject::alloc_bigint_nursery_collecting( + majit_rlib::rbigint::mul_payloads_collecting(a, b), + ), ) } } @@ -448,7 +458,9 @@ pub extern "C" fn jit_bigint_add(a: i64, b: i64) -> pyre_object::longobject::Jit return pyre_object::longobject::encode_jit_bigint_result(a as *mut BigInt); } pyre_object::longobject::encode_jit_bigint_result( - pyre_object::longobject::alloc_bigint_nursery_collecting(&*a + &*b), + pyre_object::longobject::alloc_bigint_nursery_collecting( + majit_rlib::rbigint::add_payloads_collecting(a, b), + ), ) } } diff --git a/pyre/pyre-interpreter/src/pyframe.rs b/pyre/pyre-interpreter/src/pyframe.rs index 1c68a2eeb5f..a0b0cd80342 100644 --- a/pyre/pyre-interpreter/src/pyframe.rs +++ b/pyre/pyre-interpreter/src/pyframe.rs @@ -1220,10 +1220,12 @@ pub const FRAME_BLOCK_GC_TYPE_ID: u32 = 104; pub const GC_HEADER_SIZE: usize = majit_gc::header::GcHeader::SIZE; /// Ownership selected by the caller that decides a frame's lifetime. -/// `FrameBox::new` call frames use `OldGenGc`; tracer-private snapshots use -/// `StdAlloc` so their locals remain valid until deterministic `Drop`. +/// Normal call frames use `NurseryGc`, matching `PyFrame.__init__`'s fresh +/// `[None] * size`; frame-owned auxiliary snapshots use `OldGenGc`, and +/// tracer-private snapshots use `StdAlloc` until deterministic `Drop`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum FrameLocalsArrayAllocation { + NurseryGc, OldGenGc, StdAlloc, } @@ -1285,7 +1287,25 @@ unsafe fn alloc_frame_locals_array( fill: pyre_object::PyObjectRef, allocation: FrameLocalsArrayAllocation, ) -> *mut FixedObjectArray { - if allocation == FrameLocalsArrayAllocation::OldGenGc { + if allocation == FrameLocalsArrayAllocation::NurseryGc { + let payload = pyre_object::FIXED_ARRAY_ITEMS_OFFSET + + len * std::mem::size_of::(); + if let Some(raw) = pyre_object::gc_hook::GcAllocOutcome::from_hook( + pyre_object::gc_hook::try_gc_alloc(pyre_object::PY_OBJECT_ARRAY_GC_TYPE_ID, payload), + ) + .allocated_or_abort(payload) + { + let arr = raw as *mut FixedObjectArray; + unsafe { + (*arr).len = len; + let items = (*arr).items_mut_ptr(); + for i in 0..len { + items.add(i).write(fill); + } + } + return arr; + } + } else if allocation == FrameLocalsArrayAllocation::OldGenGc { let payload = pyre_object::FIXED_ARRAY_ITEMS_OFFSET + len * std::mem::size_of::(); let raw = pyre_object::gc_hook::try_gc_alloc_stable_raw( @@ -6116,11 +6136,12 @@ impl PyFrame { // slot is the only thing that keeps it off the next sweep. It joins the // bracket the call inputs already opened: its lifetime is the rest of // this function body, so it needs no owner of its own. - let _ = pyre_object::gc_roots::pin_root(locals_cells_stack_w as PyObjectRef); + let locals_idx = _roots.publish(&[locals_cells_stack_w as PyObjectRef]); + _roots.normalize(locals_idx, 1); { // Populate the freshly-allocated array via its mutable slice. - let arr = unsafe { &mut *locals_cells_stack_w }; + let arr = unsafe { &mut *(_roots.get(locals_idx) as *mut FixedObjectArray) }; // Bind positional arguments directly -- no intermediate Vec. let nargs = args.len().min(num_locals); @@ -6128,10 +6149,9 @@ impl PyFrame { arr[i] = _roots.get(args_base + i); } - // CPython 3.11+ `co_localsplusnames` unified slot layout: - // each cellvar that ALSO appears in varnames shares its + // Each cellvar that also appears in varnames shares its // varname slot (MAKE_CELL wraps the local). Only cellvars - // NOT in varnames take a fresh slot in the cell region. + // not in varnames take a fresh slot in the cell region. // Allocating cells for the overlap would shift freevar // indices and break LOAD_DEREF on `def repeat(n): def // wrap(fn): def inner(): return (n, fn)` style closures. @@ -6141,11 +6161,13 @@ impl PyFrame { let family = unsafe { crate::pycode::w_code_cell_family(_roots.get(root_base), num_locals + i) }; + let arr = unsafe { &mut *(_roots.get(locals_idx) as *mut FixedObjectArray) }; arr[num_locals + i] = pyre_object::w_cell_new(PY_NULL, family); } let closure = _roots.get(root_base + 2); if !closure.is_null() { let nfreevars = code_ref.freevars.len(); + let arr = unsafe { &mut *(_roots.get(locals_idx) as *mut FixedObjectArray) }; for i in 0..nfreevars { let cell = unsafe { w_tuple_getitem(closure, i as i64).unwrap() }; arr[num_locals + npure + i] = cell; @@ -6154,8 +6176,9 @@ impl PyFrame { } // Stable frame-locals arrays are filled before their owning frame is - // published. `w_cell_new` uses the non-collecting old-gen allocator; - // remember the completed array before the next allocating operation. + // published. Reload after the cell allocations: the pin slot is the + // only root until the frame stores the array. + let locals_cells_stack_w = _roots.get(locals_idx) as *mut FixedObjectArray; remember_frame_locals_array(locals_cells_stack_w); let frame_stores_global = unsafe { @@ -6503,7 +6526,7 @@ pub fn createframe_obj( execution_context, )); let locals_cells_stack_w = - unsafe { alloc_frame_locals_array(size, PY_NULL, FrameLocalsArrayAllocation::OldGenGc) }; + unsafe { alloc_frame_locals_array(size, PY_NULL, FrameLocalsArrayAllocation::NurseryGc) }; let frame = PyFrame { ob_header: frame_ob_header(), pycode: _roots.get(root_base) as *const (), diff --git a/pyre/pyre-jit-trace/src/frame_layout.rs b/pyre/pyre-jit-trace/src/frame_layout.rs index a5bd7e74019..0a66199acea 100644 --- a/pyre/pyre-jit-trace/src/frame_layout.rs +++ b/pyre/pyre-jit-trace/src/frame_layout.rs @@ -140,6 +140,15 @@ unsafe extern "C" fn pyre_clear_vable_token(obj_ptr: i64) { /// to dispatch GETFIELD/SETFIELD to `InstancePtrInfo` / `StructPtrInfo`. pub fn build_pyframe_virtualizable_info() -> std::sync::Arc { let mut info = crate::virtualizable_gen::build_virtualizable_info(); + // virtualizable.py `VirtualizableInfo.__init__` obtains the array item + // descriptor from `cpu.arraydescrof(getattr(VTYPE, name).TO)`. Reuse the + // same GcArray(PyObjectRef) descriptor as ordinary frame construction and + // materialization; the macro-generated layout-only descriptor is merely a + // bootstrap value. Descriptor identity is observable to OptHeap: using + // two descriptors for `locals_cells_stack_w` prevents it from recognizing + // the unchanged slots written by `gen_store_back_in_vable` and leaves a + // full array writeback in recursive bridges. + info.replace_array_descrs(vec![crate::state::pyobject_gcarray_descr()]); // rpython/jit/metainterp/virtualizable.py `clear_vable_ptr` // + `clear_vable_descr`. The descr must carry // EffectInfo.MOST_GENERAL + OopSpecIndex.JitForceVirtualizable @@ -162,6 +171,17 @@ pub fn build_pyframe_virtualizable_info() -> std::sync::Arc { #[cfg(test)] mod tests { use super::build_pyframe_virtualizable_info; + + #[test] + fn pyframe_vable_reuses_the_gcarray_descriptor() { + let info = build_pyframe_virtualizable_info(); + let canonical = crate::state::pyobject_gcarray_descr(); + assert_eq!(info.array_item_descr(0).index(), canonical.index()); + assert!(std::sync::Arc::ptr_eq( + &info.array_item_descr(0), + &canonical + )); + } use super::{PYFRAME_VABLE_TOKEN_OFFSET, pyre_clear_vable_token}; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/pyre/pyre-jit-trace/src/helpers.rs b/pyre/pyre-jit-trace/src/helpers.rs index 2c711c7cc23..1585c587498 100644 --- a/pyre/pyre-jit-trace/src/helpers.rs +++ b/pyre/pyre-jit-trace/src/helpers.rs @@ -680,8 +680,7 @@ pub fn emit_box_int_inline( // entirely ("ignore the operation completely -- instead, it's done by // 'new'"). rewrite.py handle_malloc_operation emits the vtable // setfield via fielddescr_vtable during GC rewrite of NEW_WITH_VTABLE. - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); - ctx.heap_cache_mut().new_object(new_op); + let new_op = ctx.execute_new_with_vtable(size_descr); // Emit: SetfieldGc(v, intval, raw_int) let intval_idx = intval_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, raw_int], intval_descr); @@ -708,8 +707,7 @@ pub fn emit_box_long_inline( size_descr: majit_ir::DescrRef, value_descr: majit_ir::DescrRef, ) -> OpRef { - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); - ctx.heap_cache_mut().new_object(new_op); + let new_op = ctx.execute_new_with_vtable(size_descr); let value_idx = value_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, bigint_ref], value_descr); ctx.heapcache_setfield_cached(new_op, value_idx, bigint_ref); @@ -735,8 +733,7 @@ pub fn emit_exception_new_inline( ) -> OpRef { let (size_descr, kind_descr, w_class_descr, args_w_descr) = crate::descr::w_exception_descrs(kind); - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); - ctx.heap_cache_mut().new_object(new_op); + let new_op = ctx.execute_new_with_vtable(size_descr); let kind_const = ctx.const_int(kind as u8 as i64); let kind_idx = kind_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, kind_const], kind_descr); @@ -1105,8 +1102,7 @@ pub fn emit_object_list_inline(ctx: &mut TraceCtx, items: &[OpRef]) -> OpRef { } // Step 3 — allocate the W_ListObject wrapper. - let list = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], w_list_size_descr()); - ctx.heap_cache_mut().new_object(list); + let list = ctx.execute_new_with_vtable(w_list_size_descr()); // Step 4 — length / items / strategy SetfieldGc, mirroring the // Object-strategy arm of `w_list_new`. @@ -1160,8 +1156,7 @@ pub fn emit_empty_list_inline(ctx: &mut TraceCtx) -> OpRef { list_int_items_len_descr, list_length_descr, list_strategy_descr, w_list_size_descr, }; - let list = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], w_list_size_descr()); - ctx.heap_cache_mut().new_object(list); + let list = ctx.execute_new_with_vtable(w_list_size_descr()); let zero = ctx.const_int(0); let length_descr = list_length_descr(); @@ -1328,8 +1323,7 @@ pub fn emit_typed_list_inline( } // Step 3 — allocate the W_ListObject wrapper. - let list = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], w_list_size_descr()); - ctx.heap_cache_mut().new_object(list); + let list = ctx.execute_new_with_vtable(w_list_size_descr()); // Step 4 — initialize every scalar field, then install the active typed // storage. The pointer fields are cleared by the GC rewriter. @@ -1518,8 +1512,7 @@ pub fn emit_box_slice_inline( w_stop_descr: majit_ir::DescrRef, w_step_descr: majit_ir::DescrRef, ) -> OpRef { - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); - ctx.heap_cache_mut().new_object(new_op); + let new_op = ctx.execute_new_with_vtable(size_descr); let w_start_idx = w_start_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, w_start], w_start_descr); ctx.heapcache_setfield_cached(new_op, w_start_idx, w_start); @@ -1540,8 +1533,7 @@ pub fn emit_box_float_inline( floatval_descr: majit_ir::DescrRef, ) -> OpRef { // jtransform.py:908-911 parity: typeptr setfield filtered in trace. - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); - ctx.heap_cache_mut().new_object(new_op); + let new_op = ctx.execute_new_with_vtable(size_descr); let floatval_idx = floatval_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, raw_float], floatval_descr); ctx.heapcache_setfield_cached(new_op, floatval_idx, raw_float); @@ -1705,8 +1697,7 @@ pub fn emit_new_pyframe_inline_with_params( ctx.heapcache_setarrayitem(locals_array, idx, heapcache_item_descr_index, cell); } - let new_frame = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], pyframe_size_descr()); - ctx.heap_cache_mut().new_object(new_frame); + let new_frame = ctx.execute_new_with_vtable(pyframe_size_descr()); let code_descr = pyframe_code_descr(); let code_idx = code_descr.index(); @@ -1837,8 +1828,7 @@ pub fn emit_new_pyframe_inline_self_recursive( // Step 4 — allocate the new PyFrame. The GC tags it with // `PYFRAME_GC_TYPE_ID` because the size descr's parent type id is // registered in `pyre-jit/src/eval.rs`. - let new_frame = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], pyframe_size_descr()); - ctx.heap_cache_mut().new_object(new_frame); + let new_frame = ctx.execute_new_with_vtable(pyframe_size_descr()); // Step 5 — SetfieldGc for the constructor-visible fields, mirroring // the explicit assignments inside `new_for_call_with_closure`. diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index d476f7c8138..1159f8624fd 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -577,18 +577,8 @@ pub fn dispatch_via_miframe( // remaining Python frames unwind interpreted, exactly as they // do when the raise surfaces from a residual call. // - // The store-back belongs to that same shape and is what makes - // the unwind readable: this frame keeps running in the - // interpreter, which reads its locals out of - // `locals_cells_stack_w`, while the walk held them in the - // virtualizable boxes. `virtualizable.py write_boxes` - // writes them on every force with no way to decline, and - // `record_top_level_application_traceback` above only performs - // it concretely, for the recording pass — so without the - // emitted store-back the compiled bridge leaves the frame - // holding whatever its entry wrote, and a `tb_frame.f_locals` - // or `sys._getframe()` on the way out reads every - // post-entry local as unbound. + // `compile_exit_frame_with_exception` uses the same lazy + // virtualizable-token protocol as an ordinary return. if !recording_raise_keeps_existing_traceback(&mut wc, position) { record_top_level_application_traceback( &mut wc, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index 85617456e77..4a649c6d2de 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -2896,18 +2896,29 @@ pub(crate) fn fbw_ensure_boxed_for_ca( /// FBW-native port of `MIFrame::store_token_in_vable` (`pyjitpl.py`). /// Records `FORCE_TOKEN` + `SETFIELD_GC(vbox, token, vable_token_descr)` /// via `store_token_in_vable_setfield` and, when that fires, the -/// `GUARD_NOT_FORCED_2` with resumedata captured through the walker's own -/// single-frame snapshot machinery (`walker_capture_snapshot_for_last_guard`) -/// — the same resume coordinate (`entry_py_pc` / `outer_active_boxes`) every -/// other FBW guard uses, since pyre's blackhole can only re-enter the outer -/// Python opcode boundary. No-op when there is no standard virtualizable. +/// `GUARD_NOT_FORCED_2` with the current vable/vref state. Upstream +/// `finishframe` / `finishframe_exception` have popped every MIFrame before +/// `compile_done_with_this_frame` / `compile_exit_frame_with_exception` call +/// this: `capture_resumedata` reaches `create_empty_top_snapshot`. There is +/// no instruction to restart and no opcode-entry stack to restore. In +/// particular, ordinary guard capture must not replace the final last_instr +/// with a `py_pc - 1` resume coordinate. No-op without a standard virtualizable. pub(crate) fn fbw_store_token_in_vable( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, ) -> Result<(), DispatchError> { if ctx.trace_ctx.store_token_in_vable_setfield() { + if !ctx.trace_ctx.vable_snapshot_buildable() { + return Err(DispatchError::GuardSnapshotVableUntyped { pc: op_pc }); + } ctx.trace_ctx.record_guard(OpCode::GuardNotForced2, &[], 0); - walker_capture_snapshot_for_last_guard(ctx, op_pc)?; + let (vable_boxes, vref_boxes) = ctx.trace_ctx.build_snapshot_vable_vref_boxes(); + ctx.trace_ctx + .capture_snapshot_for_last_guard_multi_frame_with_vable_vref( + &[], + &vable_boxes, + &vref_boxes, + ); } Ok(()) } @@ -2934,8 +2945,8 @@ pub(crate) fn fbw_terminate_with_finish( /// Void variant of [`fbw_terminate_with_finish`] for the top-level /// `void_return/` portal exit (`compile_done_with_this_frame`'s VOID -/// branch, pyjitpl.py). Publishes the return coordinate and stores the -/// virtualizable back the same way, then stashes a `Type::Void`-marked payload so +/// branch, pyjitpl.py). Publishes the return coordinate, arms the same lazy +/// virtualizable token, then stashes a `Type::Void`-marked payload so /// [`crate::trace::full_body_walk_trace`] builds a `TraceAction::Finish` /// with no args (`done_with_this_frame_descr_from_types(&[])` resolves the /// void descr). Like the value path it does NOT record the `FINISH` op — diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 3ebe178b72a..b9c766944f6 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -3630,8 +3630,8 @@ fn walker_ec_enter( /// from `pyframe.py execute_frame` rather than from `enter` / `leave`. /// Whatever inlines the callee inlines those `ec.gettrace()` reads with it, so /// upstream's trace carries a guard on them and a tracer installed later cannot -/// be missed. This walker inlines neither, so an inlined callee reports -/// nothing for as long as the loop stays compiled. +/// be missed. The walker records the same reads via `record_gettrace_promote` +/// at the enter / leave sites. /// /// What keeps the omission sound is that there is no inlined callee to lose /// events for while a hook is installed: @@ -5241,6 +5241,24 @@ fn try_walker_inline_resolved_user_call_inner( .unwrap_or(FBW_DEFAULT_MAX_INLINE_RECURSION); let inline_recursion_count = fbw_inline_recursion_count(ctx, callee_code_key); let recursive_portal_present = fbw_recursive_portal_present(ctx, callee_code_key); + // The first recording has no procedure token, so it still inlines the + // `fib(2)` shape. A later *bridge* of that same compiled portal must + // not unroll the left spine (`bridge_subwalk.rs`: that continuation + // is CALL_ASSEMBLER, not a fresh peel). A different hot loop that + // calls an already-compiled recursive helper is not a bridge of that + // helper and still inlines (`recursion_past_unroll_bound_from_loop`). + if recursive_portal_present + && ctx.trace_ctx.is_bridge_trace + && crate::driver::try_driver_pair().is_some_and(|(driver, _)| { + driver + .meta_interp() + .warm_state_ref() + .get_cell_for_key(&callee_green_key) + .is_some_and(|cell| cell.is_compiled()) + }) + { + return resolved_inline_decline(op.pc, line!()); + } if inline_recursion_count >= max_unroll_recursion { if let Some((driver, _)) = crate::driver::try_driver_pair() { driver @@ -5290,8 +5308,9 @@ fn try_walker_inline_resolved_user_call_inner( // `ec.call_trace(self)` and `ec.return_trace(self, w_exitvalue)`, and // `executioncontext.py leave` adds `_trace(frame, 'leaveframe', ...)` when // a profiler is installed. `walker_ec_enter` / `walker_ec_leave` port the - // frame-chain half of that bracket and nothing else, and the walker has no - // route to record `_trace` — it calls back into app-level Python with the + // frame-chain half; `record_gettrace_promote` records the `gettrace()` + // reads those two hooks start with. The walker still has no route to + // record `_trace` itself — it calls back into app-level Python with the // callee frame as an argument. So while a hook is installed there is no // shape of this inline that can report what the callee owes, and the // answer is the one `codewriter/policy.py look_inside_graph` gives for a @@ -5471,8 +5490,12 @@ fn try_walker_inline_resolved_user_call_inner( // this frame reg (see the `*_vable_via_metainterp` short-circuits). // `u16::MAX` for a non-portal callee keeps the strict predicate // byte-identical (`inline_resolvable_seeded_frame_op` declines). + // Own-frame red, not the caller's `metadata.portal_frame_reg`. + // `built_as_portal` records the Portal *input shape* on every drained + // per-code jitcode; do not require it here. A missing filter left + // non-portal-shaped callees on `u16::MAX` so leftover-empty GETFIELD + // the portal and never saw the New `_compile` box. let callee_portal_frame_reg = crate::state::ensure_jitcode_index(callee_code_key as *const ()) - .filter(|&jc| crate::state::built_as_portal_at(jc)) .map(|jc| crate::state::portal_red_regs_at(jc).0) .unwrap_or(u16::MAX); let strict_inlinable = @@ -6765,6 +6788,7 @@ fn try_walker_inline_resolved_user_call_inner( }; callee_regs_r.set(frame_reg as usize, callee_frame); + ctx.trace_ctx.set_inline_vable_box(callee_frame); // `perform_call` creates one concrete frame per MIFrame before // `setup_call` installs the argument boxes (pyjitpl.py, // 1862-1874). Mirror that recording-time object. `setup_call` @@ -7039,6 +7063,11 @@ fn try_walker_inline_resolved_user_call_inner( ca_concrete_frame, concrete_ec, ); + // `execute_frame.call_trace` — `gettrace()` — sits between + // `enter` and `dispatch`. Snapshot failure still has to reach + // the matching `leave` below, so a recording miss here drops + // the pin rather than unwinding past the vref. + let _ = super::record_gettrace_promote(ctx, op.pc); // This inlined level is an activation `execute_frame` would have // charged the recursion counter for. Counting it at RUN time is // what a recorded call would do, and that is exactly wrong here: a @@ -7196,6 +7225,9 @@ fn try_walker_inline_resolved_user_call_inner( .registers_r .get(callee_portal_frame_reg as usize) .expect("ref register in range"); + if shadow.frame_box != OpRef::NONE { + sub_wc.trace_ctx.set_inline_vable_box(shadow.frame_box); + } } if !try_multiframe { let mut state = sub_wc.frame_state.borrow_mut(); @@ -7598,6 +7630,9 @@ fn try_walker_inline_resolved_user_call_inner( // permanently `mark_as_escaped` the caller and force a vref that never // needed forcing. let got_exception = matches!(callee_outcome, Ok((DispatchOutcome::SubRaise { .. }, _))); + // `execute_frame.return_trace` — a second `gettrace()` — sits + // between `dispatch` and `leave`. Same finally pairing as enter. + let _ = super::record_gettrace_promote(ctx, op.pc); walker_ec_leave( ctx.trace_ctx, ca_callee_frame, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 848de457975..3119ca31d24 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -2229,9 +2229,9 @@ fn create_segmented_trace( census_record("SegmentTrace::LatchRefused"); return Ok(None); } - // The latch answers for the snapshot-array stack source; this leg publishes - // from the walker mirror, whose own height check runs only in the adopter. - // Ask it here, while a refusal is still free. + // The latch already preflighted the synchronized snapshot-array stack used + // at this post-step boundary. Only require the single-frame image which + // this leg can adopt; an opcode-entry mirror describes an earlier state. // // The refusal has to unstage what the latch just wrote, and from BOTH slots: // `latch_abort_blackhole` routes an inline sub-walk to the multi-frame one, @@ -11884,11 +11884,28 @@ fn record_portal_tracefunc_guard( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, ) -> Result<(), DispatchError> { - // An inlined callee's own header is not the portal loop the compiled code - // re-enters, and the sub-walk's boxes describe the callee frame. + // An inlined callee records `gettrace` at `walker_ec_enter` / + // `walker_ec_leave` (`execute_frame.call_trace` / `return_trace`). + // This pin is the portal dispatch loop's counterpart, for the + // top-level frame whose `execute_frame` sits outside the portal. if ctx.fbw_mode.inline_subwalk { return Ok(()); } + record_gettrace_promote(ctx, op_pc) +} + +/// `executioncontext.py gettrace`: `return jit.promote(self.w_tracefunc)`. +/// +/// `call_trace` / `return_trace` both start with this read. The slot is +/// `w_tracefunc?`, so the pin is a `QUASIIMMUT_FIELD` marker plus +/// `GUARD_NOT_INVALIDATED`; `settrace` invalidates the watchers. The +/// `GuardIsnull` is the `promote(None)` half. The read is registered +/// with the heapcache, so a run of calls with no intervening store +/// collapses to one loop-invariant read. +pub(crate) fn record_gettrace_promote( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, +) -> Result<(), DispatchError> { let ec = pyre_interpreter::call::getexecutioncontext(); if ec.is_null() || !unsafe { (*ec).w_tracefunc }.is_null() { return Ok(()); @@ -11905,10 +11922,6 @@ fn record_portal_tracefunc_guard( { return Ok(()); } - // `executioncontext.py gettrace`: `return jit.promote(self.w_tracefunc)` - // on a `w_tracefunc?` slot. The marker plus `GUARD_NOT_INVALIDATED` - // is what `?` costs; `settrace` invalidates the watchers. The - // `GuardIsnull` is the `promote(None)` half this portal records. crate::state::record_quasiimmut_field(ctx.trace_ctx, ec_box, descr.clone()); walker_flush_guard_not_invalidated(ctx, op_pc)?; let read = ctx @@ -12944,9 +12957,6 @@ fn handle( w_class as pyre_object::PyObjectRef; } } - // `class_now_known` takes the vtable address: pyre tracks the - // concrete class pointer where upstream only raises HF_KNOWN_CLASS. - let known_class = descr.as_size_descr().map(|size| size.vtable() as i64); // pyjitpl.py `execute_new_with_vtable`. ctx.trace_ctx .profiler() @@ -12955,15 +12965,7 @@ fn handle( OpCode::NewWithVtable, majit_metainterp::counters::RECORDED_OPS, ); - let resbox = ctx - .trace_ctx - .record_op_with_descr(OpCode::NewWithVtable, &[], descr); - ctx.trace_ctx.heap_cache_mut().new_object(resbox); - if let Some(class) = known_class { - ctx.trace_ctx - .heap_cache_mut() - .class_now_known(resbox, class); - } + let resbox = ctx.trace_ctx.execute_new_with_vtable(descr); let dst = code[op.pc + 3] as usize; if let Some(value) = concrete { ctx.trace_ctx.set_opref_concrete(resbox, value); @@ -13765,8 +13767,9 @@ fn handle( record_portal_debugdata_guard(ctx, op.pc)?; // `execute_frame`'s `ec.call_trace` / `ec.return_trace` // (pyframe.py) read the global trace function on every call the - // loop makes. The walker records neither for an inlined callee, - // so the loop pins the slot instead. + // loop makes. Inlined callees record those reads at enter/leave; + // the top-level portal starts after `call_trace`, so this pin + // is the dispatch-loop counterpart for a hook installed later. record_portal_tracefunc_guard(ctx, op.pc)?; record_portal_profilefunc_guard(ctx, op.pc)?; diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index a80b9cbf5e0..45dfbd0ee71 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -7121,6 +7121,9 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // Range GET_ITER: virtualize exact machine-word `range` into the same // `W_IntRangeIterator` shape PyPy's inlined `descr_iter` would trace. + if foldable_runtime_helper == majit_ir::RuntimeHelperKind::GetIter { + ctx.trace_ctx.note_getiter_iterable(&r_args); + } if ctx.is_authoritative_executor && foldable_runtime_helper == majit_ir::RuntimeHelperKind::GetIter { @@ -7156,6 +7159,15 @@ pub(crate) fn dispatch_residual_call_iRd_kind( if ctx.is_authoritative_executor && foldable_runtime_helper == majit_ir::RuntimeHelperKind::ForIterNext { + // pip `_compile` traces FOR_ITER as this helper, not GetIter. + // The iterator arg is the getarrayitem_vable TOS box; record its + // mint index so leftover-empty GETFIELDs that slot, not a frame. + if r_args.first().is_some_and(|&iter_op| { + walker_concrete_ref_object(ctx, iter_op) + .is_some_and(|obj| unsafe { pyre_object::is_list_iter(obj) }) + }) { + ctx.trace_ctx.note_getiter_iterable(&r_args); + } if let Some(item_op) = spec_gate(SpecFold::ForIterNext, || { try_walker_specialize_for_iter_next(ctx, op.pc, &r_args, dst, dst_bank) })? { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index de903244393..5e6191404c2 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -20367,10 +20367,7 @@ pub(crate) fn try_walker_specialize_get_iter( &pyre_object::functional::RANGE_ITER_STEP_ONE_TYPE as *const _ as i64, ) }; - let new = ctx - .trace_ctx - .record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); - ctx.trace_ctx.heap_cache_mut().new_object(new); + let new = ctx.trace_ctx.execute_new_with_vtable(size_descr); // `stop` is `start + length` rather than the range's own stop: a promoted // step is one, so the two agree over any non-empty span, and an empty or diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs index f88e5304d0b..c385f8ebdf2 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs @@ -620,6 +620,99 @@ fn test_outer_resume_jitcode_index() -> u32 { index } +#[test] +fn branch_guard_snapshot_rechecks_the_condition_before_either_arm() { + // `guarded_branch_core` captures at `other_target` (the not-taken + // arm), not `goto_if_not`'s orgpc. A depth-0 branch resumes past + // `POP_JUMP_IF_*`; stamping the guard pc would re-run the branch + // and desync the decoded box layout. The condition stays live so + // a later shared-snapshot guard can still see it. + let live = crate::state::op_live(); + let goto = insns_opname_to_byte()["goto_if_not/iL"]; + // pc 3: goto_if_not; fall-through/taken is 7, not-taken target is 11. + let code = vec![live, 0, 0, goto, 0, 11, 0, live, 4, 0, 0, live, 7, 0]; + let runtime_jc = majit_metainterp::jitcode::JitCode::new("branch_orgpc_test"); + runtime_jc.set_body(majit_translate::jitcode::JitCodeBody { + code: code.clone(), + c_num_regs_i: 1, + startpoints: Some([0_usize, 3, 7, 11].into_iter().collect()), + ..Default::default() + }); + let mut insns = indexmap::IndexMap::new(); + insns.insert("live/".to_string(), live); + insns.insert("goto_if_not/iL".to_string(), goto); + // At orgpc the condition is live; at both arms it is dead. + crate::assembler::publish_state(&insns, &[1, 0, 0, 1, 0, 0, 0, 0, 0, 0], 10, 3); + let mut pyjit = crate::PyJitCode::skeleton(std::ptr::null()); + pyjit.jitcode = std::sync::Arc::new(runtime_jc); + pyjit.metadata.is_drained = true; + pyjit.metadata.n_py_instrs = 2; + pyjit.metadata.forward_py_pc_marker_by_jit_pc = vec![(0, 0), (7, 1), (11, 1)]; + pyjit.metadata.forward_py_pc_pred_by_jit_pc = vec![(0, 0), (7, 1), (11, 1)]; + pyjit.metadata.resume_marker_marker_by_jit_pc = + vec![(0, Some(0)), (7, Some(7)), (11, Some(11))]; + pyjit.metadata.resume_marker_pred_by_jit_pc = vec![(0, Some(0)), (7, Some(7)), (11, Some(11))]; + let installed = crate::state::install_jitcode_for(std::ptr::null(), std::sync::Arc::new(pyjit)) + as *const crate::state::JitCode; + let mut sym = crate::state::PyreSym::new_uninit(OpRef::NONE); + sym.jitcode = installed; + let mut mode = test_fbw_mode(); + mode.snapshot_sym = &sym; + let session = std::cell::RefCell::new(WalkSession::default()); + let mut tc = TraceCtx::for_test_types(&[Type::Int]); + let condbox = OpRef::input_arg_int(0); + let mut wc = WalkContext { + frame_state: WalkFrameState::new(WalkFrameStateData { + callee_shadow: None, + concrete_registers_r: Vec::new(), + outer_active_boxes: Vec::new(), + vstack_boxes: Vec::new(), + vstack_last_ref: OpRef::NONE, + vstack_reorder_saved: None, + ..Default::default() + }), + inline_callee_consts: None, + inline_poison_pcs: None, + fbw_mode: mode, + session: &session, + registers_r: &RegisterBank::default(), + registers_i: &RegisterBank::new([condbox]), + registers_f: &RegisterBank::default(), + concrete_registers_i: &mut [], + descr_refs: &[], + raw_descrs: RawDescrPool::Global, + is_authoritative_executor: false, + trace_ctx: &mut tc, + is_top_level: true, + sub_jitcode_lookup: &no_sub_jitcodes, + entry_py_pc: EntryPyPc::Py(0), + outer_resume_marker_jit_pc: Some(0), + outer_jitcode_index: unsafe { (*installed).index as u32 }, + pending_guard_snapshot_error: None, + vstack_depth: 0, + vstack_cur_pypc: 0, + vstack_valid: false, + vstack_reorder_ceiling: u32::MAX, + vstack_handler_landing_py: None, + live_before_jit_pc: 0, + live_after_jit_pc: usize::MAX, + }; + let op = decode_op_at(&code, 3).unwrap(); + goto_if_not_branch_on(&code, &op, &mut wc, condbox, 1, 11).unwrap(); + drop(wc); + let guard = tc.ops().last().unwrap(); + let snapshot = tc.get_snapshot(guard.rd_resume_position()).unwrap(); + assert_eq!( + snapshot.frames[0].pc, 11, + "resume must enter the not-taken arm" + ); + assert_eq!( + snapshot.frames[0].boxes.len(), + 0, + "the condition is dead on the not-taken arm" + ); +} + #[test] fn after_residual_guard_uses_trailing_live_before_fallthrough_twin() { let int_add = *insns_opname_to_byte() @@ -6106,7 +6199,7 @@ fn step_through_raise_records_outermost_finish_and_terminates() { } #[test] -fn top_level_raise_settles_the_vable_token() { +fn top_level_raise_arms_the_lazy_vable_token() { // `pyjitpl.py compile_exit_frame_with_exception` opens with // `store_token_in_vable()`, the same as `compile_done_with_this_frame`. // The exit therefore leaves a lazy, armed token rather than eagerly @@ -6115,6 +6208,7 @@ fn top_level_raise_settles_the_vable_token() { .get("raise/r") .expect("`raise/r` must be in insns table"); let code = [raise_byte, 0x02]; + let outer_jitcode_index = test_outer_resume_jitcode_index(); let mut tc = fresh_trace_ctx(); let mut vable_buf = vec![0u8; 65536]; bind_fake_vable(&mut tc, &mut vable_buf); @@ -6149,7 +6243,6 @@ fn top_level_raise_settles_the_vable_token() { entry_py_pc: EntryPyPc::Py(0), outer_resume_marker_jit_pc: None, outer_jitcode_index: 0, - pending_guard_snapshot_error: None, vstack_depth: 0, diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 44c41f9ae43..713f25586f6 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -15388,7 +15388,7 @@ pub(crate) fn setup_reconstructed_callee_frame( w_globals, execution_context, PY_NULL, - pyre_interpreter::pyframe::FrameLocalsArrayAllocation::OldGenGc, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::NurseryGc, ), ); drop(arg_roots); @@ -15461,7 +15461,7 @@ pub(crate) fn setup_reconstructed_callee_frame( current_globals, execution_context, current_closure, - pyre_interpreter::pyframe::FrameLocalsArrayAllocation::OldGenGc, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::NurseryGc, ), ); // The `drop(concrete_frame)` below relinquishes only the host handle for a diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 47a8df694bf..10cfe4efc17 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -954,7 +954,7 @@ fn try_commit_midbody_abort_inner( w_globals, ec, pyre_object::PY_NULL, - pyre_interpreter::pyframe::FrameLocalsArrayAllocation::OldGenGc, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::NurseryGc, ) { Ok(frame) => frame, Err(_) => return Err(MidBodyDecline::BeforeRun("callee frame allocation failed")), @@ -990,11 +990,9 @@ fn try_commit_midbody_abort_inner( }; locals_w_mut!(frame).as_mut_slice()[stack_base + rel] = *value; } - // The array is old-gen from birth (`FrameLocalsArrayAllocation::OldGenGc`) - // and `FrameLocalsRoot` only forwards the field slot, not the items: the - // young refs just stored need the remembered set to survive the boxing - // allocations below, and each minor consumes the entry, so re-arm after - // every batch that follows a possible collection. + // `PyFrame.__init__` creates this as a fresh nursery array. The barrier + // helper is harmless for that case and still covers the old-gen spill arm + // if a full nursery made `malloc_fast` reserve the block there. crate::state::frame_array_write_barrier( frame.as_mut_ptr() as *mut u8, locals_w_mut!(frame) as *mut _, @@ -2915,11 +2913,14 @@ fn try_adopt_single_frame_blackhole( // was never written at all — it still holds the pre-walk stack (see // `capture_frame_stack_from_mirror`). Take the walker's OpRef mirror, // which the latch resolved while the concrete side tables were still - // live. The segment cut stops at a boundary but at the merge point, - // where the array is equally unrefreshed, so it reads the mirror too. + // live. SegmentTrace, like ABORT_TOO_LONG, stops after the step: + // pyjitpl.py _create_segmented_trace_and_blackhole converts the current + // MIFrame and synchronized virtualizable, not the preceding opcode's + // entry stack. In particular a STORE_FAST before the merge point has + // already popped its value; restoring its entry mirror would undo that + // pop without undoing the MIFrame's instruction pointer. let takes_mirror = commit_leg == WalkEndCommitLeg::WalkAbort - || commit_leg == WalkEndCommitLeg::VableEscape - || commit_leg == WalkEndCommitLeg::SegmentTrace; + || commit_leg == WalkEndCommitLeg::VableEscape; if crate::jitcode_dispatch::fbw_debug_abort_enabled() { let from_array = crate::state::capture_frame_stack_for_publish(cf_addr, vable_frame) .map(|stack| stack.roots_snapshot()); diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index 0c80b20818c..5ff2a78eaeb 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -406,7 +406,7 @@ fn alloc_callee_frame( w_globals, execution_context, PY_NULL, - pyre_interpreter::pyframe::FrameLocalsArrayAllocation::OldGenGc, + pyre_interpreter::pyframe::FrameLocalsArrayAllocation::NurseryGc, ), ) .into_raw(); @@ -585,11 +585,22 @@ pub(crate) fn publish_residual_call_exception(exc_obj: i64) { // the blackhole) end up reading the value's `ExcKind` tag through a match // with no wildcard arm; see `exit_frame_exception_ref`. let obj = exc_obj as PyObjectRef; + // Leftover `NewWithVtable` allocates the exception in the nursery. A + // later collection that does not rewrite this native copy leaves a + // forwarding stub: the first payload word is the new address + // (`GcHeader::set_forwarding_address`), so classifying the leftover + // reads that address as `ob_type` and rejects a live ValueError. + let obj = if obj.is_null() { + obj + } else { + pyre_object::gc_hook::try_gc_current_object_address(obj as *mut u8) as PyObjectRef + }; if !obj.is_null() && unsafe { pyre_object::interp_exceptions::w_exception_kind_checked(obj) }.is_none() { reject_non_exception_channel_value(obj, "publish_residual_call_exception", String::new); } + let exc_obj = obj as i64; majit_metainterp::blackhole::BH_LAST_EXC_VALUE.with(|c| c.set(exc_obj)); store_jit_exception(exc_obj); } @@ -632,12 +643,22 @@ fn park_residual_call_exception() -> ParkedResidualException { let scope = pyre_object::gc_roots::push_roots(); let save = pyre_object::gc_roots::shadow_stack_len(); let bh_pinned = bh != 0; + let backend_pinned = backend != 0; + // Publish both cells before any normalize: `pin_root` queries after the + // first write and a collection there would move the still-unrooted one. + let mut parked = [pyre_object::PY_NULL; 2]; + let mut n = 0; if bh_pinned { - let _ = pyre_object::gc_roots::pin_root(bh as pyre_object::PyObjectRef); + parked[n] = bh as pyre_object::PyObjectRef; + n += 1; } - let backend_pinned = backend != 0; if backend_pinned { - let _ = pyre_object::gc_roots::pin_root(backend as pyre_object::PyObjectRef); + parked[n] = backend as pyre_object::PyObjectRef; + n += 1; + } + if n > 0 { + let base = scope.publish(&parked[..n]); + scope.normalize(base, n); } majit_metainterp::blackhole::BH_LAST_EXC_VALUE.with(|c| c.set(0)); drain_backend_jit_exc(); @@ -963,10 +984,10 @@ pub(crate) extern "C" fn record_inline_traceback_for_recording( let w_code = w_code_value as PyObjectRef; let w_globals = w_globals_value as PyObjectRef; let _roots = pyre_object::gc_roots::push_roots(); - let base = pyre_object::gc_roots::pin_roots(&[w_exc, w_code]); - let w_exc = pyre_object::gc_roots::shadow_stack_get(base); - let w_code = pyre_object::gc_roots::shadow_stack_get(base + 1); - let w_globals = pyre_object::gc_roots::pin_root(w_globals); + let base = _roots.pin_roots(&[w_exc, w_code, w_globals]); + let w_exc = _roots.get(base); + let w_code = _roots.get(base + 1); + let w_globals = _roots.get(base + 2); // `record_application_traceback` requires the traceback's own frame // identity. The recording walker cannot force the optimizer's virtual // locals, so materialize a traceback-only frame from the promoted callee @@ -1045,10 +1066,10 @@ pub(crate) extern "C" fn record_discarded_level_traceback( let w_globals = unsafe { pyre_interpreter::w_code_get_w_globals(w_code) }; let w_exc = exc_value as PyObjectRef; let _roots = pyre_object::gc_roots::push_roots(); - let base = pyre_object::gc_roots::pin_roots(&[w_exc, w_code]); - let w_exc = pyre_object::gc_roots::shadow_stack_get(base); - let w_code = pyre_object::gc_roots::shadow_stack_get(base + 1); - let w_globals = pyre_object::gc_roots::pin_root(w_globals); + let base = _roots.pin_roots(&[w_exc, w_code, w_globals]); + let w_exc = _roots.get(base); + let w_code = _roots.get(base + 1); + let w_globals = _roots.get(base + 2); let Ok(mut frame) = pyre_interpreter::createframe_obj( w_code as *const (), w_globals, @@ -2105,10 +2126,16 @@ fn jit_blackhole_resume_from_guard( // in eval.rs. We do this BEFORE setting up resume state so deep // recursion through the blackhole interpreter cannot accumulate // further damage. - if let Err(exc) = pyre_interpreter::stack_check::drain_jit_pending_exception() { - // Stash for the eval loop to surface — same channel the - // blackhole/force callbacks already use for cross-FFI errors. - crate::call_jit::set_pending_ca_exception(exc); + if let Err(mut exc) = pyre_interpreter::stack_check::drain_jit_pending_exception() { + // This callback returns as the result of CALL_ASSEMBLER, whose caller + // immediately executes GUARD_NO_EXCEPTION. Publish into the same two + // exception cells as every raising residual call; merely stashing the + // PyError for the outer eval loop would let the null call result reach + // bytecode consumers before that boundary (and, on AArch64, crash). + let exc_obj = exc.to_exc_object(); + if exc_obj != pyre_object::PY_NULL { + publish_residual_call_exception(exc_obj as i64); + } pyre_jit_trace::jitcode_dispatch::fbw_finish_concrete_reset(); return None; } @@ -2236,17 +2263,18 @@ fn jit_blackhole_resume_from_guard( // instead of resuming the no-exception continuation with a NULL // result. // compile.py `ResumeGuardForcedDescr.handle_fail` fishes the - // cache `handle_async_forcing` saved; no other `handle_fail` does. - // `fail_values[0]` IS the callee's `PyFrame*` for the Python portal - // (the entry-green-key recovery above relies on the same contract), so - // it names the frame a force would have attached its cache to. - let all_virtuals = if descr_arc.is_guard_forced() { - crate::eval::take_forced_virtuals_for_frame( - fail0 as *const pyre_interpreter::pyframe::PyFrame, - ) + // cache `handle_async_forcing` saved via `cpu.get_savedata_ref(deadframe)`. + let savedata = unsafe { (*deadframe).jf_savedata }; + let savedata_slot = [savedata as i64]; + let _savedata_root = unsafe { + majit_metainterp::resume::DeadFrameRefRoots::enter(&savedata_slot, |_| savedata != 0) + }; + let all_virtuals = if descr_arc.is_guard_forced() && savedata != 0 { + majit_metainterp::allvirtuals::reveal(majit_ir::GcRef(savedata_slot[0] as usize)) } else { None }; + let identity_override = (fail0 != 0).then_some(fail0); let result = blackhole_resume_via_rd_numb( &storage.rd_numb, storage.rd_consts(), @@ -2256,8 +2284,9 @@ fn jit_blackhole_resume_from_guard( Some(deadframe_types.as_slice()), guard_exc, false, // CALL_ASSEMBLER portal is jd0 (virtualizable) + identity_override, all_virtuals, - None, // `raw_deadframe` is rooted only by the copy made inside + None, ); return handle_blackhole_result(result, actual_green_key); } @@ -2431,6 +2460,9 @@ fn exit_frame_exception_ref( /// /// Never returns: the caller is about to classify the value by its `ExcKind` /// tag, and there is no correct classification for a value that has no tag. +/// Name lookups through `ob_type` / `w_class` are intentionally omitted — +/// an aligned word is not a live type, and following one is how this +/// reporter SIGSEGVed after already printing the header dump. fn reject_non_exception_channel_value( obj: PyObjectRef, site: &str, @@ -2467,23 +2499,15 @@ fn reject_non_exception_channel_value( pyre_interpreter::host_seam::emit_stderr( format!("[jit][BUG] {site}: context: {}\n", context()).as_bytes(), ); - // `words[0]` is `ob_type` and `words[1]` is `w_class`; only read through - // either when the pointer has the shape of one. `ob_type` names the - // built-in layout ("object" for every instance of a Python class), so the - // `w_class` name is the one that identifies the value. - let type_name = if words[0] != 0 && words[0].is_multiple_of(8) { - unsafe { pyre_object::pyobject::type_name_of(obj) } - } else { - "" - }; - let class_name = if words[1] != 0 && words[1].is_multiple_of(8) { - unsafe { pyre_object::w_type_get_name(words[1] as PyObjectRef).to_string() } - } else { - "".to_string() - }; + // Do not follow `ob_type` / `w_class`. Alignment is not a type proof — + // a reused nursery word that happens to be 8-aligned still faults + // `type_name_of` / `w_type_get_name`, which is how this reporter turned + // a classification abort into SIGSEGV on macos cranelift. panic!( "{site}: exception channel value is not a W_BaseException \ - (obj={obj:p} tag_byte={tag} type={type_name} class={class_name})" + (obj={obj:p} tag_byte={tag} \ + words=[{:#018x} {:#018x} {:#018x} {:#018x}])", + words[0], words[1], words[2], words[3] ); } @@ -2671,6 +2695,10 @@ pub fn blackhole_resume_via_rd_numb<'df>( // consume a phantom vable and dereference garbage; a novable resume passes // `None` for both the vinfo and the per-frame virtualizable handle. novable: bool, + // Live portal / callee PyFrame when the vable identity item decoded to + // empty (`NULLREF` after a cut remapped the snapshot box, or a TAGBOX + // whose failarg slot was never stored). `None` for novable resumes. + identity_override: Option, // compile.py — the cache `handle_async_forcing` already // materialized, when this resume is the GUARD_NOT_FORCED that follows a // force. `None` for every other guard. @@ -2815,7 +2843,7 @@ pub fn blackhole_resume_via_rd_numb<'df>( Some(vrefinfo_dyn), // resume.py:1314 metainterp_sd.virtualref_info vinfo_arg, // resume.py:1312 self.jitdriver_sd.virtualizable_info None, // resume.py:1316 greenfield_info unused in pyre - None, // heap PyFrame identity remains the live TAGBOX + identity_override, // live portal frame when the encoded identity is empty all_virtuals, // resume.py:1373-1374 GUARD_NOT_FORCED cache &allocator, ) @@ -4718,6 +4746,7 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 raw_values: Vec, guard_value_operand: Option, guard_exc: i64, + savedata: Option, }, } let outcome = { @@ -4770,6 +4799,7 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 }) .collect(); let guard_exc = backend.grab_exc_value(&frame).0 as i64; + let savedata = backend.get_savedata_ref(&frame); Outcome::Deopt { descr_arc, green_key, @@ -4777,6 +4807,7 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 raw_values, guard_value_operand, guard_exc, + savedata, } } }; @@ -4800,6 +4831,7 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 raw_values, guard_value_operand, mut guard_exc, + savedata, } => { // `grab_exc_value` cleared the only root for the pending exception // (dynasm `ca_helper`, llmodel.py:240); root the bare carrier while @@ -4813,6 +4845,16 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 exit_layout.is_traced_ref_slot(index) }) }; + // compile.py ResumeGuardForcedDescr.handle_fail reads + // `cpu.get_savedata_ref(deadframe)` after the bridge attempt. + // `dead_frame_from_ran_frame` already copied `jf_savedata`; + // root that copy across the same window. + let savedata_slot = [savedata.map_or(0, majit_ir::GcRef::as_usize) as i64]; + let _savedata_root = unsafe { + majit_metainterp::resume::DeadFrameRefRoots::enter(&savedata_slot, |_| { + savedata.is_some() + }) + }; let attempt = try_compile_ca_bridge(&descr_arc, &raw_values, guard_value_operand); if attempt.terminal_declined { // This target cannot reach compiled steady state: each CA @@ -4852,20 +4894,18 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 { return result; } - // compile.py `ResumeGuardForcedDescr.handle_fail`: only a - // GUARD_NOT_FORCED failure fishes the cache the force saved, keyed - // by the callee frame `raw_values[0]` names. - let forced_cache_owner = if descr_arc.is_guard_forced() { - callee_frame as *const pyre_interpreter::pyframe::PyFrame - } else { - std::ptr::null() - }; + let savedata = savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)); let bh = crate::eval::resume_in_blackhole_from_exit_layout( &raw_values, &exit_layout, guard_exc, - forced_cache_owner, + descr_arc.is_guard_forced().then_some(savedata).flatten(), false, + // `frame_ptr` is the compiled JITFRAME, not a PyFrame. + // CA failargs put the callee PyFrame at slot 0; that is + // the only sound identity override when the encoded + // vable identity is empty. + Some(callee_frame as i64).filter(|&ptr| ptr != 0), ); handle_blackhole_result(bh, green_key).unwrap_or(0) } @@ -5771,10 +5811,10 @@ fn bh_call_fn_impl(callable: PyObjectRef, null_or_self: PyObjectRef, args: &[PyO && unsafe { pyre_interpreter::builtin_code_get_fast_natural_arity(code) as usize } == positional_count; if exact_fixed_arity { - let _ = _roots.pin_root(code); - let _ = _roots.pin_root(receiver); - let code_slot = root_base + 2 + args.len(); - let receiver_slot = code_slot + 1; + let extra = _roots.publish(&[code, receiver]); + _roots.normalize(extra, 2); + let code_slot = extra; + let receiver_slot = extra + 1; let mut call_args = [pyre_object::PY_NULL; 4]; call_args[0] = _roots.get(receiver_slot); for (index, slot) in call_args[1..positional_count].iter_mut().enumerate() { diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index cde38d876a6..1f24c0c50fc 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -4916,6 +4916,12 @@ fn build_gc() -> Box { pyre_interpreter::active_subclass_range_hierarchy(), "GC rclass.OBJECT registration order must match the shared subclass-range census", ); + // compile.py `AllVirtuals`: the cache saved by + // ResumeGuardForcedDescr.handle_async_forcing is a real GC object held by + // JITFRAME.jf_savedata. Register it after the public object hierarchy so + // adding this frontend-private struct cannot perturb PyType type ids. + let all_virtuals_tid = gc.register_type(majit_metainterp::allvirtuals::type_info()); + majit_metainterp::allvirtuals::set_type_id(all_virtuals_tid); gc.freeze_types(); pyre_interpreter::typedef::init_subclass_ranges(); assert_subclass_ranges( @@ -5180,9 +5186,6 @@ fn install_gc_root_walkers() { majit_gc::shadow_stack::register_young_owner_reconciler( pyre_interpreter::objspace::std::mapdict::reconcile_young_owner_entries, ); - // `MetaInterp::forced_virtuals` is the same shape but lives in one mutator's - // `JIT_DRIVER` rather than a global table, so it registers per mutator - // instead — see `forced_virtuals_pruner_area`. } fn register_thread_root_areas() { @@ -5265,14 +5268,6 @@ fn register_thread_root_areas() { jit_driver, "compile_snapshot", ); - register( - forced_virtuals_root_walker_area, - jit_driver, - "forced_virtuals", - ); - // The ephemeron half of the walker above, on the same `data` so the - // prune reaches exactly the drivers the root walk reaches. - majit_gc::shadow_stack::register_mutator_pruner(forced_virtuals_pruner_area, jit_driver); } } @@ -5740,7 +5735,7 @@ unsafe extern "C" fn force_pyframe_vref( if majit_metainterp::majit_log_enabled() { eprintln!("[jit][force-hook] vref token=0x{token:x}"); } - driver.force_virtualizable_token(token); + driver.force_virtualizable_token(token, None); }) }; // `virtualref.py:174-176` — `token == TOKEN_NONE` with no `forced` means @@ -5827,7 +5822,7 @@ unsafe extern "C" fn force_pyframe(frame: *mut pyre_interpreter::PyFrame) { if majit_metainterp::majit_log_enabled() { eprintln!("[jit][force-hook] frame token=0x{token:x} frame={ptr:p}"); } - driver.force_virtualizable_token(token); + driver.force_virtualizable_token(token, Some(ptr as i64)); }); }; // Force the traced frame only when the frame handed to Python belongs @@ -5929,42 +5924,6 @@ unsafe fn compile_snapshot_root_walker_area( } } -/// GC walker for the virtual caches `handle_async_forcing` produced and left -/// for the `GUARD_NOT_FORCED` that follows. Upstream traces them through the -/// deadframe's `jf_savedata` GCREF field; pyre holds them on `MetaInterp` and -/// needs the edge drawn explicitly. -/// See `MetaInterp::walk_forced_virtuals_refs`. -unsafe fn forced_virtuals_root_walker_area( - data: *const (), - visitor: &mut dyn FnMut(&mut majit_ir::GcRef), -) { - if let Some(pair) = unsafe { jit_driver_pair_from_root_area(data) } { - pair.0.walk_forced_virtuals_refs(visitor); - } -} - -/// Drop forced-virtual caches whose owner frame the major collection is about -/// to sweep — the ephemeron half of rooting them at all. -/// -/// The force runs inside a residual `CALL_MAY_FORCE`, and two paths leave the -/// entry unconsumed: an escaped virtualizable raises instead of failing a -/// guard, and `handle_fail`'s bridge-compiled arm returns without resuming. -/// Both would otherwise pin the materialized virtuals for the process lifetime -/// and leave a key a recycled `PyFrame` address could match. -/// -/// Registered per mutator, next to `forced_virtuals_root_walker_area` and with -/// the same `data`, so the prune reaches every driver the root walk reaches. The -/// global `register_ephemeron_pruner` cannot: the table lives in this thread's -/// `JIT_DRIVER`, and a major driven by another thread would leave it pinned. -unsafe fn forced_virtuals_pruner_area( - data: *const (), - classify: &mut dyn FnMut(usize) -> Option, -) { - if let Some(pair) = unsafe { jit_driver_pair_from_root_area(data) } { - pair.0.prune_forced_virtuals(classify); - } -} - /// Re-derives the thread-local `JitDriverPair` for a GC root walk from the /// registered `JIT_DRIVER` cell pointer. /// @@ -7707,8 +7666,9 @@ fn drive_unpack_iterable_trace( .expect("a guard exit carrying resume storage carries its layout"), guard_exc, // jd1 is novable: it has no virtualizable to force. - std::ptr::null(), + None, true, + None, ); match bh { // Merge point reached: re-enter the compiled drain. @@ -7965,6 +7925,33 @@ fn drive_unpack_iterable_trace( } } +unsafe extern "C" fn leftover_is_listiter(p: *const u8) -> i32 { + if p.is_null() { + return 0; + } + unsafe { pyre_object::iterobject::is_list_iter(p as pyre_object::PyObjectRef) as i32 } +} + +/// Publish EC top (`vref_referent`, never a vref) so leftover_peel_tos can +/// find the inlined `_compile` listiter when the leftover-empty red is the +/// portal caller (TOS = ZipInfo). +fn publish_leftover_scan_frame() { + let ec = pyre_interpreter::call::getexecutioncontext(); + if ec.is_null() { + majit_metainterp::register_leftover_scan_frame(std::ptr::null()); + return; + } + let raw = unsafe { (*ec).topframeref }; + let top = pyre_interpreter::executioncontext::vref_referent(raw); + if top.is_null() + || unsafe { majit_metainterp::virtualref::ptr_is_virtual_ref(top as *const u8) } + { + majit_metainterp::register_leftover_scan_frame(std::ptr::null()); + return; + } + majit_metainterp::register_leftover_scan_frame(top as *const u8); +} + /// Eagerly register pyre-jit's hooks into pyre-interpreter so callers /// like `sys.settrace` see the JIT side from the very first user call, /// not only after the first JIT-eligible eval. Idempotent (the @@ -7980,6 +7967,10 @@ pub fn init_jit_hooks() { // dispatch (object crate included) can read it without depending on // the metainterp. majit_rlib::jit::install_we_are_jitted(majit_backend::we_are_jitted); + majit_metainterp::register_listiter_type_word( + &pyre_object::iterobject::LIST_ITER_TYPE as *const _ as usize, + ); + majit_metainterp::register_listiter_pred(leftover_is_listiter); // Phase A: build the GC and install it into the backend + pyre-object // hooks. Safe at boot — no interpreter state referenced. This makes // frames GC-owned even under PYRE_JIT=0 (#383). @@ -10864,7 +10855,16 @@ fn handle_fail( raw_values: &[i64], guard_exc: i64, _info: &majit_metainterp::virtualizable::VirtualizableInfo, -) -> HandleFailOutcome { + savedata: Option, +) -> (HandleFailOutcome, Option) { + // compile.py ResumeGuardForcedDescr.handle_fail keeps the deadframe + // (and `jf_savedata`) alive across the bridge decision. The native + // raw-exit path has already copied that field out, so root the copy + // before this function's GC hooks and reload the forwarded address. + let savedata_slot = [savedata.map_or(0, majit_ir::GcRef::as_usize) as i64]; + let _savedata_root = unsafe { + majit_metainterp::resume::DeadFrameRefRoots::enter(&savedata_slot, |_| savedata.is_some()) + }; // The guard exception arrives as a bare pointer whose deadframe root is // already gone, and bridge setup decodes resume data (allocating) before // `setup_bridge_sym` copies it onto the sym. Park it for the walker first. @@ -10942,7 +10942,10 @@ fn handle_fail( // The `ResumeInBlackhole` below decodes off the exit layout it was // handed, so retiring the entry here cannot starve it of slot types. driver.remove_compiled_loop(green_key); - return HandleFailOutcome::ResumeInBlackhole; + return ( + HandleFailOutcome::ResumeInBlackhole, + savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)), + ); } // A keyed failure proves this FOR_ITER site's instance-`__next__` @@ -11011,7 +11014,10 @@ fn handle_fail( // compile.py:708: bridge compiled → ContinueRunningNormally. // RPython: the bridge is attached to the guard descr; // re-entering compiled code will follow the bridge. - return HandleFailOutcome::BridgeCompiled; + return ( + HandleFailOutcome::BridgeCompiled, + savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)), + ); } crate::call_jit::BridgeResolution::Finished(cv) => { // #177: the walk ran the resumed frame forward to its @@ -11024,10 +11030,16 @@ fn handle_fail( pyre_jit_trace::state::ConcreteValue::Null => w_none(), other => other.to_pyobj(), }; - return HandleFailOutcome::BridgeFinished(v); + return ( + HandleFailOutcome::BridgeFinished(v), + savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)), + ); } crate::call_jit::BridgeResolution::FinishedException(cv) => { - return HandleFailOutcome::BridgeRaised(finish_concrete_raise_error(cv)); + return ( + HandleFailOutcome::BridgeRaised(finish_concrete_raise_error(cv)), + savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)), + ); } crate::call_jit::BridgeResolution::ResumeBlackhole => {} } @@ -11035,7 +11047,10 @@ fn handle_fail( } // compile.py:710-716 / pyjitpl.py:2906 (SwitchToBlackhole): // resume_in_blackhole(metainterp_sd, jitdriver_sd, self, deadframe) - HandleFailOutcome::ResumeInBlackhole + ( + HandleFailOutcome::ResumeInBlackhole, + savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)), + ) } /// Short tag for a `BlackholeResult` variant, for the `[bh-rd-numb]` @@ -11062,48 +11077,6 @@ fn blackhole_result_tag(r: &crate::call_jit::BlackholeResult) -> &'static str { /// guards (`optimizeopt/mod.rs`'s `store_final_boxes_in_guard`), so /// `is_guard_forced()` is the same discriminator upstream gets from the /// descr subtype. -fn forced_guard_cache_owner( - descr_arc: &std::sync::Arc, - frame: *const pyre_interpreter::PyFrame, -) -> *const pyre_interpreter::PyFrame { - if descr_arc.is_guard_forced() { - frame - } else { - std::ptr::null() - } -} - -/// `compile.py:956-957` — `hidden_all_virtuals = -/// metainterp_sd.cpu.get_savedata_ref(deadframe)`. -/// -/// Upstream reads the cache straight out of the deadframe it is resuming, -/// because `handle_fail` receives that deadframe. pyre's guard-failure path -/// surfaces the running frame instead, so the cache is keyed by the frame the -/// force ran against (`force_pyframe`) and taken back here by the same frame. -/// -/// The jitframe address is not usable as the key: the failing exit does not -/// name which of its slots holds the force token. -/// -/// A miss returns `None`, which resumes the ordinary way. Upstream instead -/// substitutes an empty `VirtualCache` (`compile.py`) and still runs -/// the reader with `resume_after_guard_not_forced == 2` — it can, because a -/// deadframe-local savedata slot cannot miss. A frame-keyed reconstruction -/// can, and skipping the vable section with an empty virtuals cache would -/// leave the resumed frame's virtuals unbound; falling back to a full decode -/// is the same work pyre did before the cache existed. -/// -// dont_look_inside: post-trace blackhole resume machinery. -#[majit_macros::dont_look_inside] -pub(crate) fn take_forced_virtuals_for_frame( - frame: *const pyre_interpreter::PyFrame, -) -> Option<(Vec, Vec)> { - if frame.is_null() { - return None; - } - let (driver, _) = driver_pair(); - driver.meta_interp_mut().take_forced_virtuals(frame as u64) -} - /// compile.py:710-716 resume_in_blackhole parity. /// /// RPython: resume_in_blackhole → blackhole_from_resumedata → @@ -11115,15 +11088,28 @@ pub(crate) fn resume_in_blackhole_from_exit_layout( raw_values: &[i64], exit_layout: &CompiledExitLayout, guard_exc: i64, - // `forced_guard_cache_owner` of the failing guard: the frame whose - // forced-virtual cache this resume may fish, null for any guard that is - // not a GUARD_NOT_FORCED. - forced_cache_owner: *const pyre_interpreter::PyFrame, + // `cpu.get_savedata_ref(deadframe)` for GUARD_NOT_FORCED; None for every + // other guard kind. + savedata: Option, // True when the failing guard belongs to a novable jitdriver (jd1 // `unpackiterable_driver`): its resume data has no vable section, so the // decode must not consume one. jd0 guards pass `false`. novable: bool, + // Live portal PyFrame for jd0. Used when the encoded vable identity is + // empty (`NULLREF` / unread failarg slot). Novable resumes pass `None`. + identity_override: Option, ) -> crate::call_jit::BlackholeResult { + // compile.py ResumeGuardForcedDescr.handle_fail keeps `deadframe` alive + // while it reads `cpu.get_savedata_ref(deadframe)` and passes the revealed + // AllVirtuals cache into resume.py. The native raw-exit adaptation has + // already copied that field out of the JITFRAME, so give the copied GCREF + // the same precise root lifetime. In particular, re-read the slot after + // entering it: a collection while the blackhole is being prepared may + // forward AllVirtuals and write the new address here. + let savedata_slot = [savedata.map_or(0, majit_ir::GcRef::as_usize) as i64]; + let _savedata_root = unsafe { + majit_metainterp::resume::DeadFrameRefRoots::enter(&savedata_slot, |_| savedata.is_some()) + }; // Same deadframe rooting as `handle_fail`: `decode_ref`'s TAGBOX arm reads // these slots after the resume construction has already allocated. The // scope is handed to `blackhole_resume_via_rd_numb` below rather than held @@ -11170,7 +11156,8 @@ pub(crate) fn resume_in_blackhole_from_exit_layout( // kind out of the self-describing deadframe+descr it was handed. // The sibling resume paths already pass this slice directly // (`jitdriver.rs`). - let all_virtuals = take_forced_virtuals_for_frame(forced_cache_owner); + let rooted_savedata = savedata.map(|_| majit_ir::GcRef(savedata_slot[0] as usize)); + let all_virtuals = rooted_savedata.and_then(majit_metainterp::allvirtuals::reveal); let result = crate::call_jit::blackhole_resume_via_rd_numb( &storage.rd_numb, storage.rd_consts(), @@ -11180,6 +11167,7 @@ pub(crate) fn resume_in_blackhole_from_exit_layout( Some(exit_layout.exit_types.as_slice()), guard_exc, novable, + identity_override, all_virtuals, Some(deadframe_roots), ); @@ -11307,7 +11295,8 @@ fn execute_assembler( info: &majit_metainterp::virtualizable::VirtualizableInfo, env: &PyreEnv, ) -> Option { - let mut frame_root = FrameRoot::new(frame); + let mut frame_root = FrameRoot::new(loop_red_frame(frame)); + publish_leftover_scan_frame(); frame_root.frame().set_last_instr_from_next_instr(entry_pc); // Convert tagged-immediate frame locals to heap `W_IntObject` before the @@ -11537,6 +11526,7 @@ fn execute_assembler( ref raw_values, ref exit_layout, guard_exc, + savedata, } => { match handle_fail( frame_root.frame(), @@ -11550,19 +11540,21 @@ fn execute_assembler( raw_values, guard_exc, info, + savedata, ) { - HandleFailOutcome::BridgeCompiled => Some(LoopResult::ContinueRunningNormally), + (HandleFailOutcome::BridgeCompiled, _) => Some(LoopResult::ContinueRunningNormally), // #177: single-frame bridge walk returned a concrete Finish. - HandleFailOutcome::BridgeFinished(v) => Some(LoopResult::Done(Ok(v))), - HandleFailOutcome::BridgeRaised(err) => Some(LoopResult::Done(Err(err))), - HandleFailOutcome::ResumeInBlackhole => { + (HandleFailOutcome::BridgeFinished(v), _) => Some(LoopResult::Done(Ok(v))), + (HandleFailOutcome::BridgeRaised(err), _) => Some(LoopResult::Done(Err(err))), + (HandleFailOutcome::ResumeInBlackhole, savedata) => { // compile.py:710-716 / pyjitpl.py:2906 SwitchToBlackhole let bh_result = resume_in_blackhole_from_exit_layout( raw_values, exit_layout, guard_exc, - forced_guard_cache_owner(descr_arc, frame_root.frame()), + descr_arc.is_guard_forced().then_some(savedata).flatten(), false, + Some(frame_root.frame() as *mut PyFrame as i64), ); publish_blackhole_frame_finished(&bh_result, frame_root.frame()); match &bh_result { @@ -11609,7 +11601,15 @@ fn compile_and_run_once( info: &majit_metainterp::virtualizable::VirtualizableInfo, env: &PyreEnv, ) -> Option { - let mut frame_root = FrameRoot::new(frame); + // Back-edge: leftover-empty GETFIELDs the red. An inlined `_compile` + // is EC top while `can_enter_jit` still holds the portal. Function + // entry's red is the callee being entered, not a deeper top frame. + let red = match start { + CompileOnceStart::BackEdge => loop_red_frame(frame), + CompileOnceStart::FunctionEntry => frame, + }; + let mut frame_root = FrameRoot::new(red); + publish_leftover_scan_frame(); let code = unsafe { &*pyre_interpreter::pyframe_get_pycode(frame_root.frame()) }; majit_metainterp::mc_diag_bump(match start { CompileOnceStart::BackEdge => 18, @@ -11804,7 +11804,8 @@ fn bound_reached( info: &majit_metainterp::virtualizable::VirtualizableInfo, env: &PyreEnv, ) -> Option { - let mut frame_root = FrameRoot::new(frame); + let mut frame_root = FrameRoot::new(loop_red_frame(frame)); + publish_leftover_scan_frame(); if majit_metainterp::majit_log_enabled() { let locals: Vec<(usize, Option)> = (0..locals_w!(frame_root.frame()).len().min(5)) .map(|i| { @@ -11904,6 +11905,7 @@ fn bound_reached( ref raw_values, ref exit_layout, guard_exc, + savedata, } = outcome { match handle_fail( @@ -11918,24 +11920,26 @@ fn bound_reached( raw_values, guard_exc, info, + savedata, ) { - HandleFailOutcome::BridgeCompiled => { + (HandleFailOutcome::BridgeCompiled, _) => { return Some(LoopResult::ContinueRunningNormally); } // #177: single-frame bridge walk returned a concrete Finish. - HandleFailOutcome::BridgeFinished(v) => { + (HandleFailOutcome::BridgeFinished(v), _) => { return Some(LoopResult::Done(Ok(v))); } - HandleFailOutcome::BridgeRaised(err) => { + (HandleFailOutcome::BridgeRaised(err), _) => { return Some(LoopResult::Done(Err(err))); } - HandleFailOutcome::ResumeInBlackhole => { + (HandleFailOutcome::ResumeInBlackhole, savedata) => { let bh_result = resume_in_blackhole_from_exit_layout( raw_values, exit_layout, guard_exc, - forced_guard_cache_owner(descr_arc, frame_root.frame()), + descr_arc.is_guard_forced().then_some(savedata).flatten(), false, + Some(frame_root.frame() as *mut PyFrame as i64), ); publish_blackhole_frame_finished(&bh_result, frame_root.frame()); match &bh_result { @@ -12217,6 +12221,7 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { ref raw_values, ref exit_layout, guard_exc, + savedata, } = outcome { match handle_fail( @@ -12231,27 +12236,29 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { raw_values, guard_exc, info, + savedata, ) { - HandleFailOutcome::BridgeCompiled => { + (HandleFailOutcome::BridgeCompiled, _) => { // Bridge compiled → ContinueRunningNormally → re-enter // compiled code which will follow the new bridge. // Fall through to eval_loop_jit below. } // #177: single-frame bridge walk returned a concrete Finish. // This site returns `Option` (not `LoopResult`). - HandleFailOutcome::BridgeFinished(v) => { + (HandleFailOutcome::BridgeFinished(v), _) => { return Some(Ok(v)); } - HandleFailOutcome::BridgeRaised(err) => { + (HandleFailOutcome::BridgeRaised(err), _) => { return Some(Err(err)); } - HandleFailOutcome::ResumeInBlackhole => { + (HandleFailOutcome::ResumeInBlackhole, savedata) => { let bh_result = resume_in_blackhole_from_exit_layout( raw_values, exit_layout, guard_exc, - forced_guard_cache_owner(descr_arc, frame_root.frame()), + descr_arc.is_guard_forced().then_some(savedata).flatten(), false, + Some(frame_root.frame() as *mut PyFrame as i64), ); publish_blackhole_frame_finished(&bh_result, frame_root.frame()); match &bh_result { @@ -14589,6 +14596,49 @@ fn replay_pending_fields( } } +/// One red frame per frame (`AGENTS.md`). leftover-empty GETFIELDs +/// `inputargs[index_of_virtualizable]`, the red `execute_assembler` passes. +/// `portal_frame_reg` aliases the outermost caller for an inlined callee, so +/// `can_enter_jit` can still hold the portal while the leftover iterator +/// lives on the inlined `_compile`. +/// +/// `topframeref` is a `jit.virtual_ref`. Do not `force_vref` here: +/// `execute_assembler` is about to run compiled code against this vable, +/// and `force_virtual` clears `TOKEN_TRACING_RESCALL`. Read the named +/// frame only (`vref_referent`); a still-virtual vref stays on the +/// dispatch red. +fn loop_red_frame(dispatch: &mut PyFrame) -> &mut PyFrame { + let dispatch_ptr = dispatch as *mut PyFrame; + let ec = pyre_interpreter::call::getexecutioncontext(); + if ec.is_null() { + return dispatch; + } + let raw = unsafe { (*ec).topframeref }; + let top = pyre_interpreter::executioncontext::vref_referent(raw); + if top.is_null() || top == dispatch_ptr { + return dispatch; + } + if unsafe { majit_metainterp::virtualref::ptr_is_virtual_ref(top as *const u8) } { + return dispatch; + } + // Same function only: recursive `_compile` inlined into itself. + // A different pycode is another activation (importlib, pip) whose + // vable layout is not this loop's leftover-empty prologue. + if unsafe { (*top).pycode } != dispatch.pycode { + return dispatch; + } + if unsafe { (*top).locals_cells_stack_w }.is_null() { + return dispatch; + } + if std::env::var_os("MAJIT_LEFTOVER").is_some() { + eprintln!( + "loop-red dispatch={dispatch_ptr:p} top={top:p} pycode={:p}", + dispatch.pycode + ); + } + unsafe { &mut *top } +} + // dont_look_inside: JIT-state construction machinery the tracer must not enter. #[majit_macros::dont_look_inside] pub(crate) fn build_jit_state( diff --git a/pyre/pyre-jit/src/lib.rs b/pyre/pyre-jit/src/lib.rs index e6b012425d8..ac69eb413bc 100644 --- a/pyre/pyre-jit/src/lib.rs +++ b/pyre/pyre-jit/src/lib.rs @@ -84,6 +84,7 @@ pub mod jit; mod trace_verify; // Re-export auto-generated trace functions from pyre-jit-trace +pub use majit_metainterp::leftover_peel_tos_i64; pub use pyre_jit_trace::jitcode_runtime::{ descr_demand_summary, descr_set_counts, descr_set_jit_stats, descr_spelling_gate_recheck_now, field_descr_identity_census_now, field_position_counts, field_position_jit_stats, diff --git a/pyre/pyre-jit/tests/blackhole_terminal_return.rs b/pyre/pyre-jit/tests/blackhole_terminal_return.rs index 047d15d17f5..7f5623ca290 100644 --- a/pyre/pyre-jit/tests/blackhole_terminal_return.rs +++ b/pyre/pyre-jit/tests/blackhole_terminal_return.rs @@ -90,6 +90,7 @@ fn terminal_ref_return_finishes_the_frame_and_leaves_its_execution_scope() { Some(&[Type::Ref]), 0, false, + Some(frame_ptr as i64), None, None, ); diff --git a/pyre/pyre-object/src/longobject.rs b/pyre/pyre-object/src/longobject.rs index 384a84ba5ae..cf453d6d99d 100644 --- a/pyre/pyre-object/src/longobject.rs +++ b/pyre/pyre-object/src/longobject.rs @@ -553,9 +553,11 @@ pub extern "C" fn jit_w_long_xor_raw(a: i64, b: i64) -> i64 { /// keeps the call's inputs the immutable bigints, so the optimizer forwards the /// field read and never reorders this elidable call ahead of the boxing /// `setfield_gc` that initializes the fresh result wrapper. Allocates the result -/// via the COLLECTING nursery (the call is a gcmap-rooted residual `CallR` -/// holding no unrooted pointer across the alloc), so dead bigints are reclaimed -/// by minor collections instead of accumulating in old-gen. Returns a freshly +/// via the COLLECTING nursery. The residual `CallR`'s gcmap roots its operand +/// payloads in the caller, and `rbigint::gc::add_payloads_collecting` restores +/// the native callee copies around `_x_add`'s digit-list allocation, matching +/// RPython's `push_roots(livevars)`. Thus dead bigints are reclaimed by minor +/// collections instead of accumulating in old-gen. Returns a freshly /// heap-allocated `*mut BigInt` (as i64). Allocates → `EF_ELIDABLE_OR_MEMORYERROR`. /// /// # Safety note: `extern "C"` over `i64`-encoded `*const BigInt`. The pointers @@ -573,7 +575,7 @@ pub extern "C" fn jit_bigint_add(a: i64, b: i64) -> i64 { if (&*b).get_sign() == 0 { return a as i64; } - alloc_bigint_nursery_collecting(&*a + &*b) as i64 + alloc_bigint_nursery_collecting(majit_rlib::rbigint::add_payloads_collecting(a, b)) as i64 } } @@ -596,7 +598,7 @@ pub extern "C" fn jit_bigint_sub(a: i64, b: i64) -> i64 { if (&*b).get_sign() == 0 { return a as i64; } - alloc_bigint_nursery_collecting(&*a - &*b) as i64 + alloc_bigint_nursery_collecting(majit_rlib::rbigint::sub_payloads_collecting(a, b)) as i64 } } @@ -612,7 +614,9 @@ pub extern "C" fn jit_bigint_sub_int_int(a: i64, b: i64) -> i64 { #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_mul(a: i64, b: i64) -> i64 { let (a, b) = (a as *const BigInt, b as *const BigInt); - unsafe { alloc_bigint_nursery_collecting(&*a * &*b) as i64 } + unsafe { + alloc_bigint_nursery_collecting(majit_rlib::rbigint::mul_payloads_collecting(a, b)) as i64 + } } /// `rbigint.mul_int_int_bigint_result` (`rpython/rlib/rbigint.py`, @@ -627,21 +631,27 @@ pub extern "C" fn jit_bigint_mul_int_int(a: i64, b: i64) -> i64 { #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_and(a: i64, b: i64) -> i64 { let (a, b) = (a as *const BigInt, b as *const BigInt); - unsafe { alloc_bigint_nursery_collecting(&*a & &*b) as i64 } + unsafe { + alloc_bigint_nursery_collecting(majit_rlib::rbigint::and_payloads_collecting(a, b)) as i64 + } } /// `rbigint.or_` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_or(a: i64, b: i64) -> i64 { let (a, b) = (a as *const BigInt, b as *const BigInt); - unsafe { alloc_bigint_nursery_collecting(&*a | &*b) as i64 } + unsafe { + alloc_bigint_nursery_collecting(majit_rlib::rbigint::or_payloads_collecting(a, b)) as i64 + } } /// `rbigint.xor_` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_xor(a: i64, b: i64) -> i64 { let (a, b) = (a as *const BigInt, b as *const BigInt); - unsafe { alloc_bigint_nursery_collecting(&*a ^ &*b) as i64 } + unsafe { + alloc_bigint_nursery_collecting(majit_rlib::rbigint::xor_payloads_collecting(a, b)) as i64 + } } /// `rbigint` comparison — returns the sign of `a <=> b` as `-1` / `0` / `1`. diff --git a/pyre/pyre-wasm/src/lib.rs b/pyre/pyre-wasm/src/lib.rs index 52cde219f8e..ccd4013bfa9 100644 --- a/pyre/pyre-wasm/src/lib.rs +++ b/pyre/pyre-wasm/src/lib.rs @@ -209,6 +209,8 @@ mod residual_host { [obj, attr, w_code_ptr, name_idx] => pyre_jit::call_jit::bh_load_method_self_fn, [namespace_ptr, w_code_ptr, frame_ptr, namei] => pyre_jit::call_jit::bh_load_global_fn, + [vable, vsd_off, array_off, ptr_off, len_off, items_off, kind] => + pyre_jit::leftover_peel_tos_i64, ] } diff --git a/pyre/pyrex/tests/bridge_carrier_depth_decline.rs b/pyre/pyrex/tests/bridge_carrier_depth_decline.rs index f63302d27be..287022c5cdb 100644 --- a/pyre/pyrex/tests/bridge_carrier_depth_decline.rs +++ b/pyre/pyrex/tests/bridge_carrier_depth_decline.rs @@ -153,13 +153,16 @@ fn the_answer_is_the_same_at_every_carrier_depth_cap() { } #[test] -fn a_recursive_carrier_bypasses_the_generic_depth_decline() { +fn a_recursive_carrier_capture_avoids_depth_decline_and_dirty_abort() { // Non-vacuity: `the_answer_is_the_same_at_every_carrier_depth_cap` would // also pass if the program stopped reaching the drain at all. The lowered - // generic cap must not decline this recursive carrier, and a successful - // multi-frame adoption proves the drain still handled it. `census_dump` - // reprints the whole map on every record, so the count is the value after - // the colon, never the number of occurrences. + // generic cap must not decline this recursive carrier. A caller image + // proves the recursive inline boundary was captured; a multi-frame + // terminal is an optional later outcome because successful frame + // virtualization can now keep the same execution out of the fallback + // adopter entirely. `census_dump` reprints the whole map on every record, + // so the count is the value after the colon, never the number of + // occurrences. let out = run(&[ ("PYRE_FBW_MULTIFRAME_DEPTH", "1"), ("PYRE_FBW_DEBUG_ABORT", "1"), @@ -181,12 +184,13 @@ fn a_recursive_carrier_bypasses_the_generic_depth_decline() { "a recursive carrier was rejected by pyre's generic depth cap\n{}", report("PYRE_FBW_MULTIFRAME_DEPTH=1 census", &out) ); - let adopted = text - .lines() - .any(|l| l.starts_with("[fbw-blackhole] adopted multi-frame terminal")); + let carrier_captured = text.lines().any(|l| { + l.starts_with("[fbw-blackhole] caller image") + || l.starts_with("[fbw-blackhole] adopted multi-frame terminal") + }); assert!( - adopted, - "the drain handled no recursive multi-frame carrier\n{}", + carrier_captured, + "the walk captured no recursive carrier boundary\n{}", report("PYRE_FBW_MULTIFRAME_DEPTH=1 census", &out) ); // The rollback the arm exists to avoid: an abort that ran effects and found diff --git a/scripts/llbc_extract.py b/scripts/llbc_extract.py index bcfad3d64d5..2f0de3fd680 100644 --- a/scripts/llbc_extract.py +++ b/scripts/llbc_extract.py @@ -2810,7 +2810,17 @@ def extract(eng: Engine, args: argparse.Namespace) -> None: ) before = parse_stamp(stamp) after = parse_stamp(current_stamp) - if any(before[field] != after[field] for field in ("closure", "external")): + # `stamp` is the skip-path snapshot (`include_closure=False` from + # #1811), so its `closure=` is `CLOSURE_UNCOMPUTED`. Treating that + # sentinel as a movement refuses every real extract. `stamp_skip_ok` + # already ignores the sentinel; keep the same rule here. `external=` + # is still hashed on both sides. + closure_moved = ( + before.get("closure") != CLOSURE_UNCOMPUTED + and before.get("closure") != after.get("closure") + ) + external_moved = before.get("external") != after.get("external") + if closure_moved or external_moved: invalidate_fingerprint(stamp_path, readfiles) forget_collected_inputs() unstamped.append(crate)