diff --git a/majit/majit-backend-wasm/src/codegen.rs b/majit/majit-backend-wasm/src/codegen.rs index b5bfe1eea38..b07d200a3d4 100644 --- a/majit/majit-backend-wasm/src/codegen.rs +++ b/majit/majit-backend-wasm/src/codegen.rs @@ -1367,7 +1367,9 @@ pub fn build_wasm_module( // home-slot writes overflow into the next arena slot. let bridge_finish_fi = guards .iter() - .find(|g| g.is_finish) + .find(|g| { + g.is_finish && !crate::failguard::meta_descr_is_exit_frame_with_exception(&g.meta_descr) + }) .map(|g| g.fail_index) .unwrap_or(0); // CA frames execute the source loop and this bridge on the same frozen diff --git a/majit/majit-backend-wasm/src/failguard.rs b/majit/majit-backend-wasm/src/failguard.rs index eb495ed9fb4..66ec3885826 100644 --- a/majit/majit-backend-wasm/src/failguard.rs +++ b/majit/majit-backend-wasm/src/failguard.rs @@ -25,6 +25,21 @@ pub struct WasmFailDescr { pub meta_descr: Option, } +/// `compile.py:658-662` ExitFrameWithExceptionDescrRef parity: whether a FINISH +/// exit is an ExitFrameWithException (the callee raised; slot 0 holds the +/// exception) rather than a DoneWithThisFrame. `is_finish` alone is true for +/// both, so the self-recursive CALL_ASSEMBLER arm must exclude the exception +/// variant when it picks the "clean callee finish" `fail_index` — an +/// ExitFrameWithException must route to `wasm_ca_resume_deopt`, which propagates +/// the exception, not be short-circuited to its output slot. +pub fn meta_descr_is_exit_frame_with_exception(meta_descr: &Option) -> bool { + meta_descr + .as_ref() + .and_then(|d| d.as_fail_descr()) + .map(|fd| fd.is_exit_frame_with_exception()) + .unwrap_or(false) +} + impl Descr for WasmFailDescr { fn index(&self) -> u32 { self.fail_index diff --git a/majit/majit-backend-wasm/src/lib.rs b/majit/majit-backend-wasm/src/lib.rs index d44cb2c6f5f..be6c8c10ede 100644 --- a/majit/majit-backend-wasm/src/lib.rs +++ b/majit/majit-backend-wasm/src/lib.rs @@ -1783,7 +1783,10 @@ impl majit_backend::Backend for WasmBackend { .fail_descrs .borrow() .iter() - .find(|descr| descr.is_finish) + .find(|descr| { + descr.is_finish + && !failguard::meta_descr_is_exit_frame_with_exception(&descr.meta_descr) + }) .map(|descr| descr.fail_index) .unwrap_or(0); // For a pending self target this is the exact map already embedded in diff --git a/majit/majit-ir/src/effectinfo.rs b/majit/majit-ir/src/effectinfo.rs index 6b8dc6be087..ec38117c0d0 100644 --- a/majit/majit-ir/src/effectinfo.rs +++ b/majit/majit-ir/src/effectinfo.rs @@ -631,6 +631,10 @@ pub enum PyreHelperKind { /// the in-flight iteration to the live frame instead of dropping it (the /// iterator advance is an irreversible side effect with no journal undo). ForIterNext, + /// `get_iter(obj)` — the GET_ITER residual (`iter(obj)`). The full-body + /// walker recognises exact machine-word `range` objects and emits the + /// virtual `W_IntRangeIterator` allocation shape directly. + GetIter, /// `store_deref_value(cell, value)` — the STORE_DEREF residual /// (`bh_store_deref_value_fn` via `cpu.store_deref_value_fn`). It mutates /// the cell's contents in place and RETURNS the slot value (`Ref`), so it diff --git a/majit/majit-metainterp/src/optimizeopt/unroll.rs b/majit/majit-metainterp/src/optimizeopt/unroll.rs index 654ab3c0548..00e7b8d8325 100644 --- a/majit/majit-metainterp/src/optimizeopt/unroll.rs +++ b/majit/majit-metainterp/src/optimizeopt/unroll.rs @@ -4819,11 +4819,24 @@ fn assemble_peeled_trace_with_jump_args( // Label position let label_pos = next_free_pos(max_pos); - let mut full_label_args: Vec = label_args - .iter() - .copied() - .filter(|arg| !is_trace_constant_ref(*arg, constants)) - .collect(); + // compile.py:327-328 splices `label_op` verbatim: the label arg list produced + // by import_state (make_inputargs) IS the loop-header contract, and the jump + // args (assemble_jump below) enumerate the SAME VirtualState positionally, so + // the two must stay arity-aligned. Constant virtual-state slots were already + // dropped at enum time (virtualstate.rs NotVirtual `position_in_notvirtuals` + // only assigned to non-LEVEL_CONSTANT leaves), so every arg present here is a + // live carried position. Do NOT re-derive the slot set from the post-hoc + // backend-constants map: a phase-2 guard postprocess may const-forward a + // still-live loop-carried box (e.g. the exhaust-guard's `remaining` proving + // `<=0 ∧ >=0 ⇒ ==0`), and dropping that label slot while the jump keeps + // rebinding it desyncs the label/jump contract and orphans the head guard's + // operand. RPython performs no such filter. + let mut full_label_args: Vec = label_args.iter().copied().collect(); + debug_assert!( + full_label_args.iter().all(|arg| !arg.is_constant()), + "base label arg is an inline-Const OpRef; LEVEL_CONSTANT virtual-state \ + slots must be dropped at make_inputargs enum time, not at assembly" + ); // Collect preamble-defined OpRefs BEFORE adding extra label args, // so we can filter out virtual remnants (removed New ops). diff --git a/pyre/pyre-jit-trace/src/descr.rs b/pyre/pyre-jit-trace/src/descr.rs index 396c4d240eb..d12cfc33c95 100644 --- a/pyre/pyre-jit-trace/src/descr.rs +++ b/pyre/pyre-jit-trace/src/descr.rs @@ -902,6 +902,45 @@ static RANGE_ITER_DESCR_GROUP: LazyLock = LazyLock::new(|| ) }); +static RANGE_DESCR_GROUP: LazyLock = LazyLock::new(|| { + build_object_descr_group_with_def_path( + std::mem::size_of::(), + pyre_object::functional::W_RANGE_GC_TYPE_ID, + &pyre_object::functional::RANGE_TYPE as *const _ as usize, + &[ + ( + "W_Range.start", + RANGE_START_OFFSET, + 8, + Type::Ref, + false, + true, + false, + ), + ( + "W_Range.step", + RANGE_STEP_OFFSET, + 8, + Type::Ref, + false, + true, + false, + ), + ( + "W_Range.length", + RANGE_LENGTH_OFFSET, + 8, + Type::Ref, + false, + true, + false, + ), + ], + "W_Range", + "functional::W_Range", + ) +}); + /// `Method` field layout — `w_function`, `w_self`, `w_class` per /// `function.rs:9-15`. All three are Ref slots; the JIT only consumes /// `w_function` (for guarding which method) and `w_self` (for recovering @@ -1689,6 +1728,7 @@ pub fn make_array_descr_with_full_id( use pyre_object::floatobject::{FLOAT_FLOATVAL_OFFSET, W_FloatObject}; use pyre_object::functional::{ RANGE_ITER_CURRENT_OFFSET, RANGE_ITER_REMAINING_OFFSET, RANGE_ITER_STEP_OFFSET, + RANGE_LENGTH_OFFSET, RANGE_START_OFFSET, RANGE_STEP_OFFSET, W_Range, }; use pyre_object::interp_exceptions::{ EXC_ARGS_W_OFFSET, EXC_KIND_COUNT, EXC_KIND_OFFSET, EXC_W_ATTR_OBJ_OFFSET, EXC_W_CAUSE_OFFSET, @@ -1763,6 +1803,21 @@ pub fn range_iter_step_descr() -> DescrRef { field_descr_from_group(&RANGE_ITER_DESCR_GROUP, 2) } +/// Field descriptor for `W_Range.start` (wrapped PyObjectRef). +pub fn range_start_descr() -> DescrRef { + field_descr_from_group(&RANGE_DESCR_GROUP, 0) +} + +/// Field descriptor for `W_Range.step` (wrapped PyObjectRef). +pub fn range_step_descr() -> DescrRef { + field_descr_from_group(&RANGE_DESCR_GROUP, 1) +} + +/// Field descriptor for `W_Range.length` (wrapped PyObjectRef). +pub fn range_length_descr() -> DescrRef { + field_descr_from_group(&RANGE_DESCR_GROUP, 2) +} + /// `Method.w_function` — the underlying function (`Function` or /// `BuiltinFunction`) bound by `getattr(obj, name)`. Marked immutable /// per `pypy/interpreter/function.py:567` `_Method._immutable_fields_`, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index fc0f78f7ef9..bddbe04dd29 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -203,6 +203,15 @@ pub fn dispatch_via_miframe( pyre_object::PY_NULL }, class_of_last_exc_is_const: sym.class_of_last_exc_is_const(), + // A guard-failure bridge resumes at the opcode boundary, so + // its first `jit_merge_point` crossing at this python-pc is + // the same op it is resuming INTO, not a loop crossing. The + // merge-point arm skips exactly that first crossing. Seeded + // only for bridge walks; a loop compile leaves it `None`. + bridge_entry_merge_pc: match (trace_ctx.is_bridge_trace, entry_py_pc) { + (true, EntryPyPc::Py(pc)) => Some(pc as usize), + _ => None, + }, ..Default::default() }, session, @@ -710,6 +719,7 @@ pub(crate) fn drive_bridge_frame_subwalk( .then_some(root_sym.last_exc_box()), current_exception_seed_concrete: root_sym.last_exc_value(), class_of_last_exc_is_const: root_sym.class_of_last_exc_is_const(), + ..Default::default() }, session, registers_r: &mut regs_r, @@ -1056,6 +1066,7 @@ pub(crate) fn drive_outer_frame_continuation( .then_some(root_sym.last_exc_box()), current_exception_seed_concrete: root_sym.last_exc_value(), class_of_last_exc_is_const: root_sym.class_of_last_exc_is_const(), + ..Default::default() }, session, registers_r: &mut regs_r, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index bbc9567b1ed..256f42967f3 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -529,6 +529,21 @@ pub struct FbwWalkMode { /// This is shared logically across recursive MIFrame walks; catch routing /// writes the proven-class state back into the caller's copy. pub class_of_last_exc_is_const: bool, + /// Python-pc of a guard-failure bridge walk's own resume coordinate. + /// `Some` only on the top-level walk of a bridge trace, `None` otherwise + /// (loop compiles and sub-walks). + /// + /// `generate_guard(resumepc=orgpc)` (`pyjitpl.py:2610-2626`) places a + /// guard's resume coordinate INSIDE the guarded opcode's implementation, + /// strictly past the dispatch-top `jit_merge_point`, so an RPython MIFrame + /// resumed from a guard never re-crosses the loop-header merge point at + /// position zero. Pyre's bridge walker instead resumes at the opcode + /// BOUNDARY and would re-cross the header immediately with an empty body, + /// closing a 0-progress no-op bridge. The merge-point arm consumes this + /// (via `take()`) to skip exactly the first crossing that lands on the + /// bridge's own resume coordinate, restoring the RPython positional + /// semantics. + pub bridge_entry_merge_pc: Option, } impl Clone for FbwWalkMode { @@ -567,6 +582,7 @@ impl Default for FbwWalkMode { current_exception_seed: None, current_exception_seed_concrete: pyre_object::PY_NULL, class_of_last_exc_is_const: false, + bridge_entry_merge_pc: None, } } } @@ -8222,6 +8238,22 @@ fn handle( // jdindex is the op's leading `c` byte (pyjitpl.py:1537 // `jdindex = ord(self.jitcode.code[orgpc+1])`). let jdindex = code[op.pc + 1] as i8 as usize; + // pyjitpl.py:2610-2626: a guard's resume coordinate + // (`resumepc=orgpc`) lies INSIDE the guarded opcode's + // implementation, past the dispatch-top `jit_merge_point`, so an + // RPython MIFrame resumed from a guard never re-crosses the + // loop-header merge point at position zero. The walker resumes a + // bridge at the opcode BOUNDARY, so its first crossing at the + // resume coordinate is the same op it is resuming INTO — not a + // loop crossing. Skip exactly once. `take()` clears on the first + // crossing regardless of pc, so a mid-body-resume bridge whose + // first crossing is a DIFFERENT header is unaffected. + if ctx.is_top_level + && ctx.trace_ctx.seen_loop_header_for_jdindex < 0 + && ctx.fbw_mode.bridge_entry_merge_pc.take() == Some(next_instr) + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } if ctx.trace_ctx.seen_loop_header_for_jdindex < 0 { // pyjitpl.py:1548 `if not any_operation: return`. if ctx.trace_ctx.num_ops() == 0 { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 25c4c6c6cfe..c4364be3b0c 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -1891,6 +1891,18 @@ pub(crate) fn dispatch_residual_call_iRd_kind( } } + // Range GET_ITER: virtualize exact machine-word `range` into the same + // `W_IntRangeIterator` shape PyPy's inlined `descr_iter` would trace. + if ctx.is_authoritative_executor + && ctx.is_full_body_walk + && ei.pyre_helper == majit_ir::PyreHelperKind::GetIter + { + if let Some(iter_op) = try_walker_specialize_get_iter(ctx, op.pc, &r_args, dst, dst_bank)? { + write_residual_call_result_to_dst(ctx, op.pc, dst, dst_bank, iter_op)?; + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + } + // Range FOR_ITER is a C-level iterator advance. Re-emit its field // updates so the opaque ForIterNext residual cannot invalidate optheap; // other iterator families retain the residual and its Python semantics. diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index f33e2c94b7c..77cf0a22db7 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -3223,6 +3223,22 @@ pub(crate) fn orthodox_list_append_commit( value: pyre_object::PyObjectRef, len_before: usize, ) -> Result<(), DispatchError> { + // `w_list_append` unboxes its `value` inside an inline sub-walk. A + // virtual range item must be materialized at that call boundary: otherwise + // the sub-walk's snapshot exports its raw payload as a loop-carried scalar, + // which makes a module-cell reload retain the trace-entry value. The + // identity ptr→int→ptr pair is the normal forcing shape: it preserves the + // live SSA Ref while making the virtual allocation observable to the + // optimizer, so the descended `plain_int_w` reads the current iteration's + // payload (as the real `w_list_append` call does). + let value_as_int = ctx.trace_ctx.record_op(OpCode::CastPtrToInt, &[value_op]); + ctx.trace_ctx + .set_opref_concrete(value_as_int, Value::Int(value as usize as i64)); + let value_op = ctx + .trace_ctx + .record_op(OpCode::CastIntToPtr, &[value_as_int]); + ctx.trace_ctx + .set_opref_concrete(value_op, Value::Ref(majit_ir::GcRef(value as usize))); // Stamp the receiver concrete (the sub-walk reads it as ref-arg 0; its // strategy switch needs the concrete receiver). ctx.trace_ctx.set_opref_concrete( @@ -4455,18 +4471,210 @@ pub(crate) fn try_walker_specialize_store_subscr( Ok(Some(())) } +/// Walker-native `GetIter` for an exact machine-word `range`. +/// +/// Emits the virtual `W_IntRangeIterator` allocation shape directly — the +/// iterator PyPy's inlined `descr_iter` would trace — so a locally consumed +/// iterator stays a removable virtual `New`. +pub(crate) fn try_walker_specialize_get_iter( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + r_args: &[OpRef], + _dst: usize, + dst_bank: char, +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor + || !ctx.is_full_body_walk + || dst_bank != 'r' + || r_args.len() != 1 + || ctx.fbw_mode.inline_subwalk + { + return Ok(None); + } + + let range_op = r_args[0]; + let Some(range_obj) = walker_concrete_ref_object(ctx, range_op) else { + return Ok(None); + }; + + let (concrete_start, concrete_step, concrete_length, concrete_mul, concrete_one_past) = unsafe { + if !pyre_object::functional::is_w_range(range_obj) + || !pyre_object::functional::is_exact_w_range(range_obj) + { + return Ok(None); + } + let (start_obj, _stop_obj, step_obj) = pyre_object::functional::w_range_fields(range_obj); + let length_obj = pyre_object::functional::w_range_length(range_obj); + if !pyre_object::is_int(start_obj) + || pyre_object::is_bool(start_obj) + || !pyre_object::is_int(step_obj) + || pyre_object::is_bool(step_obj) + || !pyre_object::is_int(length_obj) + || pyre_object::is_bool(length_obj) + { + return Ok(None); + } + let Some((start, _stop, step)) = pyre_object::functional::w_range_fields_i64(range_obj) + else { + return Ok(None); + }; + let Some(length) = pyre_object::functional::w_range_length_i64(range_obj) else { + return Ok(None); + }; + let one_past_i128 = start as i128 + length as i128 * step as i128; + let Ok(one_past) = i64::try_from(one_past_i128) else { + return Ok(None); + }; + let Some(mul) = length.checked_mul(step) else { + return Ok(None); + }; + let Some(one_past_checked) = start.checked_add(mul) else { + return Ok(None); + }; + debug_assert_eq!(one_past_checked, one_past); + (start, step, length, mul, one_past) + }; + + let range_type_addr = &pyre_object::functional::RANGE_TYPE as *const _ as i64; + if !range_op.is_constant() && !ctx.trace_ctx.heap_cache().is_class_known(range_op) { + let range_type_const = ctx.trace_ctx.const_int(range_type_addr); + ctx.trace_ctx + .record_guard(OpCode::GuardClass, &[range_op, range_type_const], 0); + walker_capture_snapshot_for_last_guard(ctx, op_pc)?; + } + ctx.trace_ctx + .heap_cache_mut() + .class_now_known(range_op, range_type_addr); + + let int_type_addr = &pyre_object::pyobject::INT_TYPE as *const _ as i64; + let int_type_const = ctx.trace_ctx.const_int(int_type_addr); + + let start_r = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + range_op, + crate::descr::range_start_descr(), + ); + if !ctx.trace_ctx.heap_cache().is_class_known(start_r) { + ctx.trace_ctx + .record_guard(OpCode::GuardClass, &[start_r, int_type_const], 0); + walker_capture_snapshot_for_last_guard(ctx, op_pc)?; + ctx.trace_ctx + .heap_cache_mut() + .class_now_known(start_r, int_type_addr); + } + let start_i = crate::state::opimpl_getfield_gc_i( + ctx.trace_ctx, + start_r, + crate::descr::int_intval_descr(), + ); + ctx.trace_ctx + .set_opref_concrete(start_i, majit_ir::Value::Int(concrete_start)); + + let step_r = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + range_op, + crate::descr::range_step_descr(), + ); + if !ctx.trace_ctx.heap_cache().is_class_known(step_r) { + ctx.trace_ctx + .record_guard(OpCode::GuardClass, &[step_r, int_type_const], 0); + walker_capture_snapshot_for_last_guard(ctx, op_pc)?; + ctx.trace_ctx + .heap_cache_mut() + .class_now_known(step_r, int_type_addr); + } + let step_i = + crate::state::opimpl_getfield_gc_i(ctx.trace_ctx, step_r, crate::descr::int_intval_descr()); + ctx.trace_ctx + .set_opref_concrete(step_i, majit_ir::Value::Int(concrete_step)); + + let length_r = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + range_op, + crate::descr::range_length_descr(), + ); + if !ctx.trace_ctx.heap_cache().is_class_known(length_r) { + ctx.trace_ctx + .record_guard(OpCode::GuardClass, &[length_r, int_type_const], 0); + walker_capture_snapshot_for_last_guard(ctx, op_pc)?; + ctx.trace_ctx + .heap_cache_mut() + .class_now_known(length_r, int_type_addr); + } + let length_i = crate::state::opimpl_getfield_gc_i( + ctx.trace_ctx, + length_r, + crate::descr::int_intval_descr(), + ); + ctx.trace_ctx + .set_opref_concrete(length_i, majit_ir::Value::Int(concrete_length)); + + let mul = ctx + .trace_ctx + .record_op(OpCode::IntMulOvf, &[length_i, step_i]); + ctx.trace_ctx + .set_opref_concrete(mul, majit_ir::Value::Int(concrete_mul)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoOverflow, &[])?; + + let one_past = ctx.trace_ctx.record_op(OpCode::IntAddOvf, &[start_i, mul]); + ctx.trace_ctx + .set_opref_concrete(one_past, majit_ir::Value::Int(concrete_one_past)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoOverflow, &[])?; + + let new = ctx.trace_ctx.record_op_with_descr( + OpCode::NewWithVtable, + &[], + crate::descr::w_range_iter_size_descr(), + ); + ctx.trace_ctx.heap_cache_mut().new_object(new); + + let current_descr = crate::descr::range_iter_current_descr(); + let current_index = current_descr.index(); + ctx.trace_ctx + .record_op_with_descr(OpCode::SetfieldGc, &[new, start_i], current_descr); + ctx.trace_ctx + .heapcache_setfield_cached(new, current_index, start_i); + + let remaining_descr = crate::descr::range_iter_remaining_descr(); + let remaining_index = remaining_descr.index(); + ctx.trace_ctx + .record_op_with_descr(OpCode::SetfieldGc, &[new, length_i], remaining_descr); + ctx.trace_ctx + .heapcache_setfield_cached(new, remaining_index, length_i); + + let step_descr = crate::descr::range_iter_step_descr(); + let step_index = step_descr.index(); + ctx.trace_ctx + .record_op_with_descr(OpCode::SetfieldGc, &[new, step_i], step_descr); + ctx.trace_ctx + .heapcache_setfield_cached(new, step_index, step_i); + + let range_iter_type_addr = &pyre_object::functional::RANGE_ITER_TYPE as *const _ as i64; + ctx.trace_ctx + .heap_cache_mut() + .class_now_known(new, range_iter_type_addr); + + let real_iter = unsafe { pyre_object::functional::w_range_iter(range_obj) }; + ctx.trace_ctx.set_opref_concrete( + new, + majit_ir::Value::Ref(majit_ir::GcRef(real_iter as usize)), + ); + ctx.vstack_last_ref = new; + + Ok(Some(new)) +} + /// Walker-native `ForIterNext` for `W_IntRangeIterator`. /// /// The generic residual advances the shared iterator before an abort can /// occur, and forward-delivery preserves that consumed item. This inline /// path keeps that deliberately irreversible advance: it never journals or /// rolls the cursor back. It instead emits the `W_IntRangeIterator.next` -/// field-update shape with a branchless continuation mask, leaving the -/// codewriter's existing trailing `GuardNonnull` to select loop exit. +/// field-update shape with a continuation guard. Its false side resumes at +/// the same FOR_ITER coordinate as the codewriter's ordinary exhaustion edge. /// -/// The Phase 1 result remains a normal `W_IntObject` allocation. Its mask is -/// only for the exhaustion result; item virtualization is intentionally not -/// attempted here. +/// The continuation item is a normal virtualizable `W_IntObject`; allocation +/// removal elides it until an escaping consumer or a deopt needs a real box. pub(crate) fn try_walker_specialize_for_iter_next( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, @@ -4549,9 +4757,33 @@ pub(crate) fn try_walker_specialize_for_iter_next( .heap_cache_mut() .class_now_known(iter_op, range_iter_type_addr); - // Emit the cursor update without a separate exhaustion guard. `continues` - // is 0/1, so the exhausted path writes the existing cursor values and the - // masked item becomes NULL for the pre-existing GuardNonnull. + if !concrete_continues { + // Exhausted arrival: the walker concretely reached remaining==0 (a nested + // inner loop run to completion inside the outer body). Record the + // routing guard for the false continue predicate, then present the + // exhaustion edge exactly as the residual does: a NULL Ref that the + // codewriter's trailing GuardNonnull consumes as the loop exit. The + // iterator is already exhausted, so no cursor advance and no in-flight + // capture. + let zero = ctx.trace_ctx.const_int(0); + let remaining = crate::state::opimpl_getfield_gc_i( + ctx.trace_ctx, + iter_op, + crate::descr::range_iter_remaining_descr(), + ); + let continues = ctx.trace_ctx.record_op(OpCode::IntGt, &[remaining, zero]); + ctx.trace_ctx.set_opref_concrete(continues, Value::Int(0)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardFalse, &[continues])?; + let null_item = ctx.trace_ctx.record_op(OpCode::CastIntToPtr, &[zero]); + ctx.trace_ctx + .set_opref_concrete(null_item, Value::Ref(majit_ir::GcRef(0))); + return Ok(Some(null_item)); + } + + // Guard the continue arm before constructing the item. The false arm + // resumes at this FOR_ITER, where the interpreter takes the existing + // exhaustion edge (iterator retained, no item pushed). This avoids the + // pointer-mask representation which forced the item to be materialized. let current = crate::state::opimpl_getfield_gc_i( ctx.trace_ctx, iter_op, @@ -4571,6 +4803,10 @@ pub(crate) fn try_walker_specialize_for_iter_next( let continues = ctx.trace_ctx.record_op(OpCode::IntGt, &[remaining, zero]); ctx.trace_ctx .set_opref_concrete(continues, Value::Int(concrete_continues as i64)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[continues])?; + + // The continue guard establishes `continues == 1` on the trace path. Keep + // Slice-1's wrapping IntAdd and live-iterator SetfieldGc updates intact. let delta = ctx.trace_ctx.record_op(OpCode::IntMul, &[step, continues]); ctx.trace_ctx.set_opref_concrete( delta, @@ -4605,52 +4841,36 @@ pub(crate) fn try_walker_specialize_for_iter_next( ctx.trace_ctx .heapcache_setfield_cached(iter_op, remaining_descr.index(), next_remaining); - // Phase 1 keeps the item boxed. Mask its pointer through an Int word so - // exhaustion produces a NULL Ref without an additional guard or side exit. - let boxed = crate::state::wrapint(ctx.trace_ctx, current); - let boxed_as_int = ctx.trace_ctx.record_op(OpCode::CastPtrToInt, &[boxed]); - let mask = ctx.trace_ctx.record_op(OpCode::IntSub, &[zero, continues]); - let mask_concrete = 0i64.wrapping_sub(concrete_continues as i64); - ctx.trace_ctx - .set_opref_concrete(mask, Value::Int(mask_concrete)); - let masked_as_int = ctx - .trace_ctx - .record_op(OpCode::IntAnd, &[boxed_as_int, mask]); - let masked_item = ctx - .trace_ctx - .record_op(OpCode::CastIntToPtr, &[masked_as_int]); + // `wrapint` is the transparent `NewWithVtable(W_IntObject)` + + // `SetfieldGc(intval=current)` shape allocation removal virtualizes. Do + // not feed it through pointer arithmetic: locally consumed items stay + // virtual, while normal forcing materializes escaping items. + let item = crate::state::wrapint(ctx.trace_ctx, current); // Tracing executes the real range cursor advance. The direct helper is // the same `W_IntRangeIterator.next` implementation used by the residual; // do not journal it, because abort recovery forwards this exact item. let concrete_item = unsafe { pyre_object::functional::w_range_iter_next(iter_obj) }; debug_assert_eq!(concrete_item.is_some(), concrete_continues); - let concrete_item_ptr = concrete_item.unwrap_or(pyre_object::PY_NULL); - let concrete_box_ptr = if concrete_continues { - concrete_item_ptr as usize - } else { - 0 - }; - ctx.trace_ctx - .set_opref_concrete(boxed, Value::Ref(majit_ir::GcRef(concrete_box_ptr))); - ctx.trace_ctx - .set_opref_concrete(boxed_as_int, Value::Int(concrete_box_ptr as i64)); - let masked_concrete = (concrete_box_ptr as i64) & mask_concrete; - ctx.trace_ctx - .set_opref_concrete(masked_as_int, Value::Int(masked_concrete)); + let concrete_item_ptr = concrete_item.expect("GuardTrue(continues) implies a range item"); ctx.trace_ctx.set_opref_concrete( - masked_item, - Value::Ref(majit_ir::GcRef(masked_concrete as usize)), + item, + Value::Ref(majit_ir::GcRef(concrete_item_ptr as usize)), ); - if concrete_continues { - fbw_foriter_inflight_capture(concrete_item_ptr, body); - // Range iteration stays at the C level, so the operand-stack mirror - // remains valid and must receive the item produced by FOR_ITER. - ctx.vstack_last_ref = masked_item; - } + // Keep the virtual payload's concrete shadow paired with the concrete New. + // A later body guard can then encode the virtual `i` in its snapshot and + // blackhole will rematerialize the right item on deopt. + ctx.trace_ctx + .set_opref_concrete(current, Value::Int(concrete_current)); + + fbw_foriter_inflight_capture(concrete_item_ptr, body); + // Range iteration stays at the C level, so the operand-stack mirror + // remains valid and must receive the item produced by FOR_ITER. Its + // virtual state is captured by subsequent body-guard snapshots. + ctx.vstack_last_ref = item; - Ok(Some(masked_item)) + Ok(Some(item)) } /// Specialize `STORE_SUBSCR target[const_slice] = source` for a same-length, diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index 03b317dda24..a570a5b32ab 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -3377,6 +3377,7 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 // values so the driver borrow is released before the blackhole re-enters it. enum Outcome { Finished(i64), + FinishedException(i64), Deopt { descr_arc: std::sync::Arc, green_key: u64, @@ -3394,7 +3395,19 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 .as_fail_descr() .expect("CA deopt: get_latest_descr_arc returned a non-FailDescr Descr"); if descr.is_finish() { - Outcome::Finished(backend.get_ref_value(&frame, 0).as_usize() as i64) + let result = backend.get_ref_value(&frame, 0).as_usize() as i64; + // compile.py:658-662 ExitFrameWithExceptionDescrRef parity: a FINISH + // descr is either DoneWithThisFrame (return the value) or + // ExitFrameWithException (the callee raised; slot 0 holds the + // exception). The self-recursive callee re-raised — propagate it + // through the exception channel like the outer Finished arm + // (eval.rs:6066) instead of banking it as the recursive-call + // return, which would surface as `int + `. + if descr.is_exit_frame_with_exception() { + Outcome::FinishedException(result) + } else { + Outcome::Finished(result) + } } else { let green_key = majit_backend::descr_owning_jct(descr) .map(|jct| jct.green_key()) @@ -3424,6 +3437,16 @@ pub extern "C" fn wasm_ca_resume_deopt(frame_ptr: i64, compiled_ptr: i64) -> i64 match outcome { Outcome::Finished(r) => r, + // warmspot.py:998-1005 ExitFrameWithExceptionRef: the callee raised. + // Publish the exception so the caller's GUARD_NO_EXCEPTION (after the + // CALL_ASSEMBLER) fires, and return garbage — parity with + // `handle_blackhole_result`'s ExitFrameWithExceptionRef arm. + Outcome::FinishedException(exc) => { + if exc != 0 { + store_jit_exception(exc); + } + 0 + } Outcome::Deopt { descr_arc, green_key, diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index aa2f3d91887..a8496e0cf76 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -9860,6 +9860,7 @@ impl CodeWriter { let iter_var = residual_call!( get_iter_fn_idx, CallFlavor::MayForce, + majit_ir::PyreHelperKind::GetIter, vec![], vec![iterable_value], vec![], diff --git a/pyre/pyre-object/src/functional.rs b/pyre/pyre-object/src/functional.rs index 2c72aeb8759..860f4e336eb 100644 --- a/pyre/pyre-object/src/functional.rs +++ b/pyre/pyre-object/src/functional.rs @@ -689,6 +689,10 @@ pub struct W_Range { pub length: PyObjectRef, } +pub const RANGE_START_OFFSET: usize = std::mem::offset_of!(W_Range, start); +pub const RANGE_STEP_OFFSET: usize = std::mem::offset_of!(W_Range, step); +pub const RANGE_LENGTH_OFFSET: usize = std::mem::offset_of!(W_Range, length); + /// Allocate a `W_Range` from three wrapped int/long bounds. `step` must /// already be non-zero (the caller raises `ValueError` for a zero step /// before reaching here). The element count is computed once here and @@ -735,6 +739,13 @@ pub unsafe fn is_w_range(obj: PyObjectRef) -> bool { unsafe { py_type_check(obj, &RANGE_TYPE) } } +/// # Safety +/// `obj` must be a valid, non-null pointer to a `PyObject`. +#[inline] +pub unsafe fn is_exact_w_range(obj: PyObjectRef) -> bool { + unsafe { std::ptr::eq((*obj).ob_type, &RANGE_TYPE) } +} + /// Read the wrapped `(start, stop, step)` triple of a range object. /// /// # Safety