diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index 53c57502220..fe4bd352d46 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -972,6 +972,61 @@ thread_local! { /// trace executes that residual once on later iterations, so the generic /// nested-replay decline does not apply to this resolved descriptor path. pub(crate) static EXCEPTION_STRING_INLINE_ACTIVE: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// Code keys of the callees a FOR_ITER body admitted under + /// [`CalleeReplaySafety::DeferredCall`], outermost first. Non-empty for + /// the lifetime of such a sub-walk ([`ForiterDeferredInlineGuard`]), which + /// is what arms the deferred-call arm of + /// [`fbw_abort_nested_unjournaled_residual`]. + static FBW_FORITER_DEFERRED_INLINE: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + /// Callee code keys whose deferred body reached a CALL residual the lever + /// could not inline. The gate declines them up front from then on, so the + /// backstop abort costs one attempt per callee instead of storming. + static FBW_FORITER_DEFERRED_DENY: std::cell::RefCell> = + std::cell::RefCell::new(std::collections::HashSet::new()); +} + +/// Marks the sub-walk of a callee admitted into a FOR_ITER body under +/// [`CalleeReplaySafety::DeferredCall`] for its whole lifetime, so a nested +/// residual the lever could not inline can recognise the admission it breaks +/// (and the callee to deny) rather than executing. +pub(crate) struct ForiterDeferredInlineGuard(bool); + +impl ForiterDeferredInlineGuard { + pub(crate) fn enter(callee_code_key: usize, deferred: bool) -> Self { + if deferred { + FBW_FORITER_DEFERRED_INLINE.with(|c| c.borrow_mut().push(callee_code_key)); + } + ForiterDeferredInlineGuard(deferred) + } +} + +impl Drop for ForiterDeferredInlineGuard { + fn drop(&mut self) { + if self.0 { + FBW_FORITER_DEFERRED_INLINE.with(|c| { + c.borrow_mut().pop(); + }); + } + } +} + +/// The outermost callee the active sub-walk was admitted for under +/// [`CalleeReplaySafety::DeferredCall`], or `None` outside such a sub-walk. +/// Declining that callee suppresses the whole nest: a body that calls another +/// is itself `DeferredCall`, so no admitted caller sits above it. +fn fbw_foriter_deferred_inline_outermost() -> Option { + FBW_FORITER_DEFERRED_INLINE.with(|c| c.borrow().first().copied()) +} + +pub(crate) fn fbw_foriter_deferred_call_denied(callee_code_key: usize) -> bool { + FBW_FORITER_DEFERRED_DENY.with(|c| c.borrow().contains(&callee_code_key)) +} + +fn fbw_foriter_deny_deferred_call(callee_code_key: usize) { + FBW_FORITER_DEFERRED_DENY.with(|c| { + c.borrow_mut().insert(callee_code_key); + }); } /// Whether the active inline sub-walk is one of the hazard classes the blanket @@ -1032,6 +1087,15 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( // nested-decline guard, which is for FOREIGN unjournaled residuals. let in_selfrec_fold = SELFREC_CA_FOLD_ACTIVE.with(|c| c.get()); let in_exception_string_inline = EXCEPTION_STRING_INLINE_ACTIVE.with(|c| c.get()); + // A FOR_ITER-body inline admitted under `CalleeReplaySafety::DeferredCall` + // stands on the promise that the sub-walk commits nothing: the static scan + // cleared every direct heap write, leaving only Python-level CALL residuals + // whose callee the lever resolves here. One that did not inline breaks the + // promise, so abort BEFORE it executes — every op the sub-walk has run so + // far is write-free, so the resume re-runs the body benignly. Denying the + // admitted callee makes the next attempt decline it statically, so this + // costs one abort per callee rather than an abort per trace attempt. + let foriter_deferred_inline = fbw_foriter_deferred_inline_outermost(); // Narrowed decline: the general depth-≥2 nested // residual inline is sound now that the portal-runner ABI is correct — a // straight-line mutating callee inlines bit-exact. Only two callee shapes @@ -1048,8 +1112,11 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( if !in_selfrec_fold && !in_exception_string_inline && !ctx.session.borrow().framestack.is_empty() - && fbw_inline_callee_hazardous(ctx) + && (foriter_deferred_inline.is_some() || fbw_inline_callee_hazardous(ctx)) { + if let Some(callee_code_key) = foriter_deferred_inline { + fbw_foriter_deny_deferred_call(callee_code_key); + } let (outer_resume, stack_overrides) = { let session = ctx.session.borrow(); match session.framestack.first().and_then(|f| f.parent.as_ref()) { @@ -1276,41 +1343,111 @@ pub(crate) fn fbw_abort_resume_py_pc( Some(python_pc_for_jitcode_pc(&jc.payload.metadata, abort_jit_pc) as usize) } +/// Every pc in `body_code` that some op can branch to: the `goto` family and +/// `catch_exception` carry their target as the label operand, and +/// `int_*_jump_if_ovf` carries an overflow target ahead of its operands. +/// +/// `None` when an op carries a label this decode cannot locate — a var-list +/// or a pyre payload ahead of the `L` — since a missed target would let a +/// freshness claim survive a join it does not hold across. +fn body_branch_targets(body_code: &[u8]) -> Option> { + let mut targets = std::collections::HashSet::new(); + let mut pc = 0usize; + while pc < body_code.len() { + let d = crate::jitcode_runtime::decode_op_at(body_code, pc)?; + if d.argcodes.contains('L') { + // Operand widths follow `decode_op_at`; only the fixed-width forms + // can precede the label, so anything else gives up. + let mut cursor = d.pc + 1; + let mut target = None; + for operand in d.argcodes.chars() { + match operand { + 'L' => { + target = Some(u16::from_le_bytes([ + *body_code.get(cursor)?, + *body_code.get(cursor + 1)?, + ]) as usize); + break; + } + 'i' | 'c' | 'r' | 'f' => cursor += 1, + 'd' | 'j' => cursor += 2, + _ => break, + } + } + targets.insert(target?); + } + pc = d.next_pc; + } + Some(targets) +} + +/// Replay safety of one inline candidate's body inside a FOR_ITER body, as +/// judged by [`fbw_callee_body_replay_safety`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum CalleeReplaySafety { + /// No op in the body can commit a live-heap effect. + Clean, + /// Clean apart from Python-level CALL residuals, whose callee is resolved + /// only at walk time. + DeferredCall, + /// Carries a live-heap effect a replay would double. + Dirty, +} + /// Whether an inline callee can be replayed from its caller's CALL boundary /// without duplicating a live-heap effect. The inline sub-walk's deopt /// snapshot does not yet carry its own callee frame, so this is deliberately -/// stricter than ordinary inlining: unknown calls and every live-heap write -/// decline up front. +/// stricter than ordinary inlining: every live-heap write declines up front. /// -/// A `new_with_vtable/d>r` result is fresh within this body. Its -/// initialization write is benign only when the target field is immutable; -/// `wrapint` is the important instance (`W_IntObject.intval`). Freshness may -/// pass through `ref_copy`, but every other Ref-producing instruction clears -/// it, so a later `setfield_gc` cannot accidentally be classified as an -/// initialization of an earlier allocation. -pub(crate) fn fbw_callee_body_side_effect_free( +/// A Python-level CALL residual is the one shape this static scan cannot +/// settle: its callee is a runtime value, so whether the sub-walk inlines it +/// (leaving nothing to replay) or executes it (which may write) is known only +/// at the call. Those bodies report [`CalleeReplaySafety::DeferredCall`] and +/// the lever decides at the call — see +/// [`fbw_abort_nested_unjournaled_residual`], which aborts before executing a +/// residual that did not inline. Every other unproven residual is `Dirty`. +/// +/// A `new_with_vtable/d>r` or `new_array*` result is fresh within this body. +/// A `setfield_gc` initialization write into one is benign only when the +/// target field is immutable (`wrapint` is the important instance, +/// `W_IntObject.intval`); a `setarrayitem_gc` into a fresh array is benign +/// outright, since replay writes the replay's own array (`BUILD_TUPLE` / +/// `BUILD_LIST` fill their backing block this way). Freshness may pass +/// through `ref_copy`, but every other Ref-producing instruction clears it, +/// so a later store cannot accidentally be classified as an initialization of +/// an earlier allocation, and every branch target drops the whole set — a +/// register reaching a join can hold whichever allocation the taken path put +/// there, which this straight-line scan cannot name. +pub(crate) fn fbw_callee_body_replay_safety( body_code: &[u8], args_all_numeric: bool, num_regs_i: usize, constants_i: &[i64], callee_descr_refs: &[DescrRef], -) -> bool { +) -> CalleeReplaySafety { + let Some(branch_targets) = body_branch_targets(body_code) else { + return CalleeReplaySafety::Dirty; + }; let mut fresh_ref_regs = [false; u8::MAX as usize + 1]; + let mut deferred_call = false; let mut pc = 0usize; while pc < body_code.len() { + if branch_targets.contains(&pc) { + fresh_ref_regs = [false; u8::MAX as usize + 1]; + } let Some(d) = crate::jitcode_runtime::decode_op_at(body_code, pc) else { - return false; + return CalleeReplaySafety::Dirty; }; if d.opname.starts_with("residual_call") { let Some(descr_index) = residual_call_descr_index_in_body(body_code, &d) else { - return false; + return CalleeReplaySafety::Dirty; }; let Some(call_descr) = callee_descr_refs .get(descr_index) .and_then(|descr| descr.as_call_descr()) else { - return false; + return CalleeReplaySafety::Dirty; }; let ei = call_descr.get_extra_info(); // `ForIterNext` is deliberately not accepted here: it advances the @@ -1319,8 +1456,23 @@ pub(crate) fn fbw_callee_body_side_effect_free( // double-consume. A FOR_ITER-bearing body is declined anyway — its // mandatory `GET_ITER` (`MayForce`) predecessor fails this scan // first — so this only removes a latent landmine, not live inlines. - let provably_side_effect_free = - ei.check_is_elidable() || ei.extraeffect == majit_ir::ExtraEffect::LoopInvariant; + // `load_const` / `load_global` / `box_int` are tagged `CanRaise` + // only to keep the `_OS_CANRAISE` invariant (effectinfo.rs); each + // is a read or a fresh allocation, so re-running one commits + // nothing to the live heap. The BUILD_TUPLE / BUILD_LIST array + // consumers are the same shape one level up: they read a + // freshly-built backing array and return a brand-new container. + let replay_safe_read = matches!( + ei.pyre_helper, + majit_ir::PyreHelperKind::LoadConst + | majit_ir::PyreHelperKind::LoadGlobal + | majit_ir::PyreHelperKind::BoxInt + | majit_ir::PyreHelperKind::NewtupleFromArray + | majit_ir::PyreHelperKind::NewlistFromArray + ); + let provably_side_effect_free = replay_safe_read + || ei.check_is_elidable() + || ei.extraeffect == majit_ir::ExtraEffect::LoopInvariant; if !provably_side_effect_free && !residual_call_is_specialized_plain_int_add( body_code, @@ -1331,13 +1483,28 @@ pub(crate) fn fbw_callee_body_side_effect_free( callee_descr_refs, ) { - return false; + // A Python-level CALL is the one shape this scan cannot + // settle: the inline lever binds its callee only at the call, + // so whether it leaves a residual behind — and what that + // residual writes — is not a property of this body. Defer it; + // the backstop aborts before executing one that did not + // inline. + if matches!( + ei.pyre_helper, + majit_ir::PyreHelperKind::CallFn + | majit_ir::PyreHelperKind::CallKw + | majit_ir::PyreHelperKind::CallFunctionEx + ) { + deferred_call = true; + } else { + return CalleeReplaySafety::Dirty; + } } } else if d.opname.starts_with("setfield_gc") { // Canonical setfield shapes are `rd`: the target ref is // operand 0 and the field descr is operand 2. let Some(&target_reg) = body_code.get(d.pc + 1) else { - return false; + return CalleeReplaySafety::Dirty; }; let descr_index = decode_descr_index(body_code, &d, 2); let immutable_field = callee_descr_refs @@ -1345,26 +1512,39 @@ pub(crate) fn fbw_callee_body_side_effect_free( .and_then(|descr| descr.as_field_descr()) .is_some_and(|field| field.is_immutable()); if !fresh_ref_regs[target_reg as usize] || !immutable_field { - return false; + return CalleeReplaySafety::Dirty; } - } else if d.opname.starts_with("setarrayitem_gc") - || d.opname.starts_with("setinteriorfield_gc") + } else if d.opname.starts_with("setarrayitem_gc") { + // The dual of the `setfield_gc` rule: a store into an array this + // body just allocated is an initialization, not a live-heap write, + // so replaying it writes the replay's own fresh array. The + // canonical shapes put the array register in operand 0 (`r…`); the + // `iiid` raw-address form carries no array register to prove fresh. + let target_fresh = d.argcodes.starts_with('r') + && body_code + .get(d.pc + 1) + .is_some_and(|reg| fresh_ref_regs[*reg as usize]); + if !target_fresh { + return CalleeReplaySafety::Dirty; + } + } else if d.opname.starts_with("setinteriorfield_gc") || d.opname.starts_with("raw_store") || d.opname.starts_with("cond_call") || d.opname.starts_with("call_assembler") || d.opname.starts_with("inline_call") { - // Array/interior/raw stores and non-residual call forms cannot be - // proven replay-safe from this single callee body. - return false; + // Interior/raw stores and non-residual call forms cannot be proven + // replay-safe from this single callee body. + return CalleeReplaySafety::Dirty; } // The result byte is always the final operand for `>r` forms. if d.argcodes.ends_with(">r") { let Some(&dst) = body_code.get(d.next_pc.saturating_sub(1)) else { - return false; + return CalleeReplaySafety::Dirty; }; fresh_ref_regs[dst as usize] = d.key == "new_with_vtable/d>r" + || d.opname.starts_with("new_array") || (d.key == "ref_copy/r>r" && body_code .get(d.pc + 1) @@ -1372,7 +1552,11 @@ pub(crate) fn fbw_callee_body_side_effect_free( } pc = d.next_pc; } - true + if deferred_call { + CalleeReplaySafety::DeferredCall + } else { + CalleeReplaySafety::Clean + } } pub(crate) fn fbw_callee_body_has_binary_op_residual( diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 54f496d295f..2acd507e8cc 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -1244,6 +1244,167 @@ pub(crate) fn reconstructed_all_ref_call_stack( stack.first().is_some_and(|c| !c.is_null()).then_some(stack) } +/// Fold a keyword call's `kwnames`->parameter permutation at trace time so a +/// `call_kw` reuses the positional inline path. The constant `kwnames` tuple +/// and the callee's static parameter names are both known at record time, so +/// the reorder is a pure trace-time permutation of the argument boxes into +/// parameter order — `_match_signature` (`@jit.unroll_safe`) unrolled and +/// folded. Once reordered the seeding is identical to a positional call. +/// +/// `r_args` layout is `[callable, self_or_null, kwnames, arg0..argN-1]`; the +/// trailing `nkw` args are the keyword values, `kwnames[j]` naming +/// `arg[n_pos + j]` where `n_pos = nargs - nkw`. +/// +/// Returns `None` (declining to the residual call, no behavior change) for any +/// shape the plain positional seeding cannot serve: a non-constant / non-tuple +/// `kwnames`, a callee with `*args` / `**kwargs` / keyword-only parameters, an +/// argument count that does not exactly fill the positional parameters (a +/// default would be needed), a keyword naming an unknown parameter, or a +/// keyword colliding with a positionally-filled parameter ("multiple values"). +/// +/// # Safety +/// `w_code` must be the live code object pointer for the resolved callable. +unsafe fn fbw_reorder_call_kw_args( + r_args: &[OpRef], + arg_concretes: &[ConcreteValue], + w_code: *const (), + nparams: usize, +) -> Option<(Vec, Vec)> { + if r_args.len() < 3 || arg_concretes.len() < 3 { + return None; + } + let ConcreteValue::Ref(kwnames) = arg_concretes[2] else { + return None; + }; + if kwnames.is_null() || !unsafe { pyre_object::is_tuple(kwnames) } { + return None; + } + let args = &r_args[3..]; + let arg_conc = &arg_concretes[3..]; + let nargs = args.len(); + if arg_conc.len() != nargs { + return None; + } + let nkw = unsafe { pyre_object::w_tuple_len(kwnames) }; + // Every positional parameter must be filled exactly once from a passed arg: + // no defaults, no *args/**kwargs/keyword-only slots the seeding would leave + // unbound. + if nparams == 0 || nkw > nargs || nargs != nparams { + return None; + } + let raw = unsafe { + pyre_interpreter::w_code_get_ptr(w_code as pyre_object::PyObjectRef) + as *const pyre_interpreter::CodeObject + }; + if raw.is_null() { + return None; + } + let flags = unsafe { (*raw).flags }; + if flags.contains(pyre_interpreter::CodeFlags::VARARGS) + || flags.contains(pyre_interpreter::CodeFlags::VARKEYWORDS) + || unsafe { (*raw).kwonlyarg_count } != 0 + { + return None; + } + let varnames = unsafe { &(*raw).varnames }; + if varnames.len() < nparams { + return None; + } + let n_pos = nargs - nkw; + let mut slot_args: Vec> = vec![None; nparams]; + let mut slot_conc: Vec> = vec![None; nparams]; + for k in 0..n_pos { + slot_args[k] = Some(args[k]); + slot_conc[k] = Some(arg_conc[k]); + } + for j in 0..nkw { + let name_obj = unsafe { pyre_object::w_tuple_getitem(kwnames, j as i64) }?; + if !unsafe { pyre_object::is_str(name_obj) } { + return None; + } + let name = unsafe { pyre_object::w_str_get_wtf8(name_obj) } + .as_str() + .ok()?; + let pi = varnames[..nparams] + .iter() + .position(|v| v.as_str() == name)?; + // A keyword may only bind a parameter past the positional fill, and each + // parameter at most once (else Python raises "multiple values for + // argument"). + if pi < n_pos || slot_args[pi].is_some() { + return None; + } + slot_args[pi] = Some(args[n_pos + j]); + slot_conc[pi] = Some(arg_conc[n_pos + j]); + } + let mut out_args = Vec::with_capacity(nparams); + let mut out_conc = Vec::with_capacity(nparams); + for k in 0..nparams { + out_args.push(slot_args[k]?); + out_conc.push(slot_conc[k]?); + } + Some((out_args, out_conc)) +} + +/// Unpack a `bh_call_function_ex_fn(callable, self_or_null, starargs, +/// kwargs_or_null)` star tuple into the positional argument boxes the inline +/// path seeds from, or `None` to leave the call a residual. +/// +/// The elements are read out of the heap cache rather than off the tuple, so +/// this folds exactly when the star tuple is virtual at the call — the +/// `args = (...)` / `f(*args)` pair the walker just recorded, whose +/// `wrappeditems` block and per-index stores are still cached +/// (`try_walker_specialize_newtuple_object`). A tuple that arrived from +/// anywhere else has no cached block and declines, as does any `**kwargs` +/// merge (the helper's `kwargs_or_null` is then a real mapping) and any arity +/// that is not the callee's exact parameter count. +fn fbw_unpack_call_function_ex_args( + ctx: &mut WalkContext<'_, '_, Sym>, + r_args: &[OpRef], + arg_concretes: &[ConcreteValue], + nparams: usize, +) -> Option<(Vec, Vec)> { + if r_args.len() < 4 || arg_concretes.len() < 4 || nparams == 0 { + return None; + } + // `kwargs_or_null` is the checked `PY_NULL` sentinel for a call with no + // `**` merge; anything else is a real mapping this fold does not bind. + match arg_concretes[3] { + ConcreteValue::Null => {} + ConcreteValue::Ref(kwargs) if kwargs.is_null() || kwargs == pyre_object::PY_NULL => {} + _ => return None, + } + let starargs = r_args[2]; + let items_descr = crate::descr::tuple_wrappeditems_descr(); + let block = ctx + .trace_ctx + .heapcache_getfield_cached(starargs, items_descr.index())?; + // The cached length pins the arity: the callee takes exactly `nparams` + // positional parameters, and a mismatch is a runtime TypeError the inline + // path does not model. + let len_op = ctx.trace_ctx.heap_cache().arraylen(block)?; + match len_op.inline_const_to_value() { + Some(majit_ir::Value::Int(n)) if n as usize == nparams => {} + _ => return None, + } + let array_descr_index = crate::state::pyobject_gcarray_descr().index(); + let mut args = Vec::with_capacity(nparams); + let mut concretes = Vec::with_capacity(nparams); + for index in 0..nparams { + let elem = ctx.trace_ctx.heapcache_getarrayitem( + block, + OpRef::ConstInt(index as i64), + array_descr_index, + )?; + // Take each concrete from the element box itself, not from the tuple, + // so the seeded shadow is the one the symbolic argument carries. + let concrete = walker_concrete_ref_object(ctx, elem)?; + args.push(elem); + concretes.push(ConcreteValue::Ref(concrete)); + } + Some((args, concretes)) +} + pub(crate) fn try_walker_inline_user_call( ctx: &mut WalkContext<'_, '_, Sym>, op: &DecodedOp, @@ -1261,19 +1422,20 @@ pub(crate) fn try_walker_inline_user_call( if !ctx.is_authoritative_executor { return Ok(None); } - // Only a genuine Python call helper (`call_fn` / `call_fn_N`, tagged - // `PyreHelperKind::CallFn` by the flatten lowering) is an inline - // target. Every container/builtin helper routed here carries a - // different tag or `None` (`store_subscr_fn` -> StoreSubscr, - // `normalize_raise_varargs_fn` / `set_current_exception` -> None). - // Without this guard `d[f] = v` with a 1-arg function key `f` lowers - // to `residual_call_r_v(store_subscr_fn, [d, f, v])`, whose ref args - // pass the function sniff below and are mis-inlined as `f(v)`, - // skipping the store. Upstream never inlines a Python call at a - // residual_call site (inlinable calls get their own inline_call - // jitcodes); this restores that invariant for the pyre FBW - // inline-at-residual lever. - if pyre_helper != majit_ir::PyreHelperKind::CallFn { + // Only a genuine Python call helper is an inline target: positional + // `call_fn` / `call_fn_N` (`PyreHelperKind::CallFn`) and keyword `call_kw_N` + // (`PyreHelperKind::CallKw`), both tagged by the flatten lowering. Every + // container/builtin helper routed here carries a different tag or `None` + // (`store_subscr_fn` -> StoreSubscr, `normalize_raise_varargs_fn` / + // `set_current_exception` -> None). Without this guard `d[f] = v` with a + // 1-arg function key `f` lowers to `residual_call_r_v(store_subscr_fn, [d, + // f, v])`, whose ref args pass the function sniff below and are mis-inlined + // as `f(v)`, skipping the store. Upstream never inlines a Python call at a + // residual_call site (inlinable calls get their own inline_call jitcodes); + // this restores that invariant for the pyre FBW inline-at-residual lever. + let is_call_kw = pyre_helper == majit_ir::PyreHelperKind::CallKw; + let is_call_function_ex = pyre_helper == majit_ir::PyreHelperKind::CallFunctionEx; + if pyre_helper != majit_ir::PyreHelperKind::CallFn && !is_call_kw && !is_call_function_ex { return Ok(None); } if r_args.is_empty() { @@ -1298,22 +1460,57 @@ pub(crate) fn try_walker_inline_user_call( if callable.is_null() { return Ok(None); } - let ConcreteValue::Ref(null_or_self) = arg_concretes[1] else { - return Ok(None); + // The receiver slot is a checked `PY_NULL` sentinel for a plain no-receiver + // call; its concrete shadow arrives as `Null` (`call_kw`) or `Ref(PY_NULL)` + // (`call_fn`). Both mean "no receiver" (`method_form = false`). + let null_or_self = match arg_concretes[1] { + ConcreteValue::Ref(r) => r, + ConcreteValue::Null => pyre_object::PY_NULL, + _ => return Ok(None), }; - let method_form = !null_or_self.is_null(); - let mut callee_args = Vec::with_capacity(r_args.len().saturating_sub(1)); - let mut callee_arg_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); - if method_form { - callee_args.push(r_args[1]); - callee_arg_concretes.push(arg_concretes[1]); - } - callee_args.extend_from_slice(&r_args[2..]); - callee_arg_concretes.extend_from_slice(&arg_concretes[2..]); + let method_form = !null_or_self.is_null() && null_or_self != pyre_object::PY_NULL; let Some((w_code, nparams, has_closure)) = (unsafe { resolve_inlinable_callee(callable) }) else { return Ok(None); }; + let (callee_args, callee_arg_concretes) = if is_call_kw { + // A keyword call folds its `kwnames`->parameter permutation at trace + // time (`fbw_reorder_call_kw_args`) so the reordered param-order args + // seed the callee exactly like a positional call. A bound-method-form + // keyword call is not yet folded. + if method_form { + return Ok(None); + } + let Some(reordered) = + (unsafe { fbw_reorder_call_kw_args(r_args, &arg_concretes, w_code, nparams) }) + else { + return Ok(None); + }; + reordered + } else if is_call_function_ex { + // `f(*args)` unpacks the star tuple into positional arguments, which is + // a trace-time reorder whenever the tuple is the one this trace just + // built: its element boxes are still in the heap cache, so the callee + // seeds from them and the tuple keeps no consumer. + if method_form { + return Ok(None); + } + let Some(unpacked) = fbw_unpack_call_function_ex_args(ctx, r_args, &arg_concretes, nparams) + else { + return Ok(None); + }; + unpacked + } else { + let mut callee_args = Vec::with_capacity(r_args.len().saturating_sub(1)); + let mut callee_arg_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); + if method_form { + callee_args.push(r_args[1]); + callee_arg_concretes.push(arg_concretes[1]); + } + callee_args.extend_from_slice(&r_args[2..]); + callee_arg_concretes.extend_from_slice(&arg_concretes[2..]); + (callee_args, callee_arg_concretes) + }; try_walker_inline_resolved_user_call( ctx, op, @@ -1470,18 +1667,35 @@ pub(crate) fn try_walker_inline_resolved_user_call( // An inline sub-walk inside a FOR_ITER body resumes a guard at the // caller's CALL boundary, so deopt re-executes the whole callee. Replaying // a live-heap mutation would double it; the nested-residual decline catches - // that only after an abort storm. A side-effect-free callee replays - // benignly, so admit it. - if fbw_foriter_inflight_active() - && !fbw_callee_body_side_effect_free( + // that only after an abort storm. A callee whose body commits nothing + // replays benignly, so admit it. + // + // A body whose only unproven ops are Python-level CALL residuals is + // admitted too: this same gate re-runs for each callee the lever resolves + // below it, and one it cannot inline aborts before executing + // (`fbw_abort_nested_unjournaled_residual`) and denies this callee, so a + // deferred body commits nothing either. Without that the whole nest + // declines — `helper(i)` calling `add(i, 1, 2)` residualizes both calls + // per iteration, though each body on its own is pure arithmetic. + let mut foriter_deferred_admit = false; + if fbw_foriter_inflight_active() { + let admit = match fbw_callee_body_replay_safety( body.code, args_all_numeric, body.num_regs_i, body.constants_i, callee_descr_refs, - ) - { - return Ok(None); + ) { + CalleeReplaySafety::Clean => true, + CalleeReplaySafety::DeferredCall => { + foriter_deferred_admit = !fbw_foriter_deferred_call_denied(callee_code_key); + foriter_deferred_admit + } + CalleeReplaySafety::Dirty => false, + }; + if !admit { + return Ok(None); + } } if method_form && !allow_method_load_attr @@ -2222,6 +2436,8 @@ pub(crate) fn try_walker_inline_resolved_user_call( // Track this callee for the lifetime of the sub-walk so nested // self-calls see the correct recursion depth. let _inline_frame = InlineFrameGuard::enter(ctx.session, callee_code_key, parent_frame); + let _foriter_deferred = + ForiterDeferredInlineGuard::enter(callee_code_key, foriter_deferred_admit); if let Some(frame) = ActiveResumeFrame::current(ctx.session, ctx.fbw_mode.snapshot_sym) { if frame.body_matches(&body) { seed_callee_vstack_mirror(&mut sub_wc, &frame); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 6155a8ea0b6..2143d5544e7 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -2575,6 +2575,20 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } + // The arities `makespecialisedtuple2` does not claim take the canonical + // array-backed `W_TupleObject` shape instead, so a non-escaping BUILD_TUPLE + // of any width folds away rather than allocating through the opaque + // residual. Reached only after the `spec_ii` fold above declines; falls + // through to the residual for any shape it cannot reproduce (SAFE — never + // declined). + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::NewtupleFromArray + && try_walker_specialize_newtuple_object(ctx, op.pc, &r_args, dst, dst_bank)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + // #171: virtualize a non-escaping BUILD_LIST (`newlist_from_array`) by // decomposing it into the `opimpl_newlist` shape (`pyjitpl.py`) — // `new_with_vtable` + `new_array` + `setarrayitem_gc` + `setfield_gc` — diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 9cc9ffb5bb2..4f97005ca34 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -2197,6 +2197,100 @@ pub(crate) fn try_walker_specialize_newlist( Ok(Some(())) } +/// FBW virtualization of the array-backed BUILD_TUPLE — the arities +/// `makespecialisedtuple2` does not claim. Sibling of +/// [`try_walker_specialize_newtuple`] (arity-2 plain-int `spec_ii`) and +/// [`try_walker_specialize_newlist`], reached only after the `spec_ii` fold +/// declines, so that path stays byte-identical. +/// +/// `lower_tuple_build_hlop_to_insn` lowers BUILD_TUPLE to `new_array_clear` + +/// per-index `setarrayitem_gc` + a `newtuple_from_array` residual. Re-emit the +/// canonical `W_TupleObject` shape walker-native (`new_with_vtable` + +/// `w_class` / `wrappeditems` `setfield_gc` over a fresh items block), reading +/// the elements straight out of the array heap-cache so the array build keeps +/// no consumer and DCEs. A tuple that never escapes then folds away entirely, +/// and one that does escape materializes from the same fields the residual +/// would have written. +/// +/// Arity 2 is `makespecialisedtuple2` territory (`Cls_ii` / `Cls_ff` / +/// `Cls_oo`, `specialisedtupleobject.py`): the runtime never builds an +/// array-backed tuple there, so emitting one would diverge from what the +/// blackhole rebuilds on deopt. Declined here — the `spec_ii` fold owns the +/// int-int case and the residual owns the rest. The empty tuple is declined +/// too (no element to recover a length from). +/// +/// Returns `Ok(Some(()))` when folded; `Ok(None)` falls through to the opaque +/// residual, which stays correct for any shape — a non-const array length or +/// an element without a concrete Ref shadow is not declined, just not folded. +pub(crate) fn try_walker_specialize_newtuple_object( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + r_args: &[OpRef], + dst: usize, + dst_bank: char, +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor || dst_bank != 'r' { + return Ok(None); + } + if r_args.len() != 1 { + return Ok(None); + } + let arr = r_args[0]; + // Const backing-array length (`new_array_clear(Const(len))` seeded + // `heapcache.arraylen`; a cleared array has every slot set, so read the + // length directly rather than probing getarrayitem until a miss). + let len = { + let Some(len_op) = ctx.trace_ctx.heap_cache().arraylen(arr) else { + return Ok(None); + }; + match len_op.inline_const_to_value() { + Some(majit_ir::Value::Int(n)) if n >= 1 => n as usize, + _ => return Ok(None), + } + }; + if len == 2 { + return Ok(None); + } + + // Element boxes the BUILD_TUPLE `setarrayitem_gc` ops stored; a cache miss + // (clobbered array / non-const index) bails to the opaque residual. + let descr_idx = crate::state::pyobject_gcarray_descr().index(); + let mut items: Vec = Vec::with_capacity(len); + for i in 0..len { + let Some(elem) = + ctx.trace_ctx + .heapcache_getarrayitem(arr, OpRef::ConstInt(i as i64), descr_idx) + else { + return Ok(None); + }; + items.push(elem); + } + let mut concretes: Vec = Vec::with_capacity(len); + for &it in &items { + let Some(obj) = walker_concrete_ref_object(ctx, it) else { + return Ok(None); + }; + concretes.push(obj); + } + + // Concrete shadow: a fresh array-backed tuple from the element shadows + // (`w_tuple_new` parity for every arity but 2). A new allocation with no + // heap mutation, safe during the walk like `wrapint`. Built before the + // emit so a failure leaves no orphan ops in the trace. + let result_concrete = pyre_object::w_tuple_new_array_backed(concretes); + if result_concrete.is_null() { + return Ok(None); + } + + let tuple_op = crate::helpers::emit_object_tuple_inline(ctx.trace_ctx, &items); + ctx.trace_ctx.set_opref_concrete( + tuple_op, + majit_ir::Value::Ref(majit_ir::GcRef(result_concrete as usize)), + ); + write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, tuple_op)?; + Ok(Some(())) +} + /// #195 / #73: FBW virtualization of an arity-2 plain-int BUILD_TUPLE. /// `lower_tuple_build_hlop_to_insn` lowers BUILD_TUPLE to `new_array_clear` /// + per-index `setarrayitem_gc` + a `newtuple_from_array` residual