From 8cd58ae18d51c52430254a185eb92889ccb11c6c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 25 Jul 2026 11:47:52 +0900 Subject: [PATCH 1/4] jit: inline keyword calls by folding the kwnames permutation at trace time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FBW inline lever admitted only `PyreHelperKind::CallFn`, so every CALL_KW site stayed a `bh_call_kw` residual that re-ran the whole keyword binding — frame allocation, signature match, kwnames binding, argument boxing — on each iteration. Admit `PyreHelperKind::CallKw` too. `fbw_reorder_call_kw_args` reads the constant kwnames tuple and the callee's `co_varnames`, both known at record time, and reorders the argument boxes into parameter order so the existing positional seeding in `try_walker_inline_resolved_user_call` serves the call unchanged. It declines to the residual for anything it cannot settle statically: a non-constant or non-str kwnames tuple, a name that matches no parameter or one already filled positionally, `*args`/`**kwargs`/kwonly callees, an argument count other than the parameter count, and the bound-method form. The receiver slot (argument index 1) arrives as `ConcreteValue::Null` for `call_kw` where `call_fn` produces `Ref(PY_NULL)`; both are the checked "no receiver" sentinel, so accept either. `synth/call_kw_hot_loop` drops from ~700ns to ~1.0ns per iteration and its compiled loop records no `call_may_force`. Assisted-by: Claude --- .../src/jitcode_dispatch/inline_call.rs | 172 +++++++++++++++--- 1 file changed, 148 insertions(+), 24 deletions(-) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 54f496d295f..b1475fb294b 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -1244,6 +1244,108 @@ pub(crate) fn reconstructed_all_ref_call_stack( stack.first().is_some_and(|c| !c.is_null()).then_some(stack) } +/// Fold a keyword call's `kwnames`->parameter permutation at trace time so a +/// `call_kw` reuses the positional inline path. The constant `kwnames` tuple +/// and the callee's static parameter names are both known at record time, so +/// the reorder is a pure trace-time permutation of the argument boxes into +/// parameter order — `_match_signature` (`@jit.unroll_safe`) unrolled and +/// folded. Once reordered the seeding is identical to a positional call. +/// +/// `r_args` layout is `[callable, self_or_null, kwnames, arg0..argN-1]`; the +/// trailing `nkw` args are the keyword values, `kwnames[j]` naming +/// `arg[n_pos + j]` where `n_pos = nargs - nkw`. +/// +/// Returns `None` (declining to the residual call, no behavior change) for any +/// shape the plain positional seeding cannot serve: a non-constant / non-tuple +/// `kwnames`, a callee with `*args` / `**kwargs` / keyword-only parameters, an +/// argument count that does not exactly fill the positional parameters (a +/// default would be needed), a keyword naming an unknown parameter, or a +/// keyword colliding with a positionally-filled parameter ("multiple values"). +/// +/// # Safety +/// `w_code` must be the live code object pointer for the resolved callable. +unsafe fn fbw_reorder_call_kw_args( + r_args: &[OpRef], + arg_concretes: &[ConcreteValue], + w_code: *const (), + nparams: usize, +) -> Option<(Vec, Vec)> { + if r_args.len() < 3 || arg_concretes.len() < 3 { + return None; + } + let ConcreteValue::Ref(kwnames) = arg_concretes[2] else { + return None; + }; + if kwnames.is_null() || !unsafe { pyre_object::is_tuple(kwnames) } { + return None; + } + let args = &r_args[3..]; + let arg_conc = &arg_concretes[3..]; + let nargs = args.len(); + if arg_conc.len() != nargs { + return None; + } + let nkw = unsafe { pyre_object::w_tuple_len(kwnames) }; + // Every positional parameter must be filled exactly once from a passed arg: + // no defaults, no *args/**kwargs/keyword-only slots the seeding would leave + // unbound. + if nparams == 0 || nkw > nargs || nargs != nparams { + return None; + } + let raw = unsafe { + pyre_interpreter::w_code_get_ptr(w_code as pyre_object::PyObjectRef) + as *const pyre_interpreter::CodeObject + }; + if raw.is_null() { + return None; + } + let flags = unsafe { (*raw).flags }; + if flags.contains(pyre_interpreter::CodeFlags::VARARGS) + || flags.contains(pyre_interpreter::CodeFlags::VARKEYWORDS) + || unsafe { (*raw).kwonlyarg_count } != 0 + { + return None; + } + let varnames = unsafe { &(*raw).varnames }; + if varnames.len() < nparams { + return None; + } + let n_pos = nargs - nkw; + let mut slot_args: Vec> = vec![None; nparams]; + let mut slot_conc: Vec> = vec![None; nparams]; + for k in 0..n_pos { + slot_args[k] = Some(args[k]); + slot_conc[k] = Some(arg_conc[k]); + } + for j in 0..nkw { + let name_obj = unsafe { pyre_object::w_tuple_getitem(kwnames, j as i64) }?; + if !unsafe { pyre_object::is_str(name_obj) } { + return None; + } + let name = unsafe { pyre_object::w_str_get_wtf8(name_obj) } + .as_str() + .ok()?; + let pi = varnames[..nparams] + .iter() + .position(|v| v.as_str() == name)?; + // A keyword may only bind a parameter past the positional fill, and each + // parameter at most once (else Python raises "multiple values for + // argument"). + if pi < n_pos || slot_args[pi].is_some() { + return None; + } + slot_args[pi] = Some(args[n_pos + j]); + slot_conc[pi] = Some(arg_conc[n_pos + j]); + } + let mut out_args = Vec::with_capacity(nparams); + let mut out_conc = Vec::with_capacity(nparams); + for k in 0..nparams { + out_args.push(slot_args[k]?); + out_conc.push(slot_conc[k]?); + } + Some((out_args, out_conc)) +} + pub(crate) fn try_walker_inline_user_call( ctx: &mut WalkContext<'_, '_, Sym>, op: &DecodedOp, @@ -1261,19 +1363,19 @@ pub(crate) fn try_walker_inline_user_call( if !ctx.is_authoritative_executor { return Ok(None); } - // Only a genuine Python call helper (`call_fn` / `call_fn_N`, tagged - // `PyreHelperKind::CallFn` by the flatten lowering) is an inline - // target. Every container/builtin helper routed here carries a - // different tag or `None` (`store_subscr_fn` -> StoreSubscr, - // `normalize_raise_varargs_fn` / `set_current_exception` -> None). - // Without this guard `d[f] = v` with a 1-arg function key `f` lowers - // to `residual_call_r_v(store_subscr_fn, [d, f, v])`, whose ref args - // pass the function sniff below and are mis-inlined as `f(v)`, - // skipping the store. Upstream never inlines a Python call at a - // residual_call site (inlinable calls get their own inline_call - // jitcodes); this restores that invariant for the pyre FBW - // inline-at-residual lever. - if pyre_helper != majit_ir::PyreHelperKind::CallFn { + // Only a genuine Python call helper is an inline target: positional + // `call_fn` / `call_fn_N` (`PyreHelperKind::CallFn`) and keyword `call_kw_N` + // (`PyreHelperKind::CallKw`), both tagged by the flatten lowering. Every + // container/builtin helper routed here carries a different tag or `None` + // (`store_subscr_fn` -> StoreSubscr, `normalize_raise_varargs_fn` / + // `set_current_exception` -> None). Without this guard `d[f] = v` with a + // 1-arg function key `f` lowers to `residual_call_r_v(store_subscr_fn, [d, + // f, v])`, whose ref args pass the function sniff below and are mis-inlined + // as `f(v)`, skipping the store. Upstream never inlines a Python call at a + // residual_call site (inlinable calls get their own inline_call jitcodes); + // this restores that invariant for the pyre FBW inline-at-residual lever. + let is_call_kw = pyre_helper == majit_ir::PyreHelperKind::CallKw; + if pyre_helper != majit_ir::PyreHelperKind::CallFn && !is_call_kw { return Ok(None); } if r_args.is_empty() { @@ -1298,22 +1400,44 @@ pub(crate) fn try_walker_inline_user_call( if callable.is_null() { return Ok(None); } - let ConcreteValue::Ref(null_or_self) = arg_concretes[1] else { - return Ok(None); + // The receiver slot is a checked `PY_NULL` sentinel for a plain no-receiver + // call; its concrete shadow arrives as `Null` (`call_kw`) or `Ref(PY_NULL)` + // (`call_fn`). Both mean "no receiver" (`method_form = false`). + let null_or_self = match arg_concretes[1] { + ConcreteValue::Ref(r) => r, + ConcreteValue::Null => pyre_object::PY_NULL, + _ => return Ok(None), }; - let method_form = !null_or_self.is_null(); - let mut callee_args = Vec::with_capacity(r_args.len().saturating_sub(1)); - let mut callee_arg_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); - if method_form { - callee_args.push(r_args[1]); - callee_arg_concretes.push(arg_concretes[1]); - } - callee_args.extend_from_slice(&r_args[2..]); - callee_arg_concretes.extend_from_slice(&arg_concretes[2..]); + let method_form = !null_or_self.is_null() && null_or_self != pyre_object::PY_NULL; let Some((w_code, nparams, has_closure)) = (unsafe { resolve_inlinable_callee(callable) }) else { return Ok(None); }; + let (callee_args, callee_arg_concretes) = if is_call_kw { + // A keyword call folds its `kwnames`->parameter permutation at trace + // time (`fbw_reorder_call_kw_args`) so the reordered param-order args + // seed the callee exactly like a positional call. A bound-method-form + // keyword call is not yet folded. + if method_form { + return Ok(None); + } + let Some(reordered) = + (unsafe { fbw_reorder_call_kw_args(r_args, &arg_concretes, w_code, nparams) }) + else { + return Ok(None); + }; + reordered + } else { + let mut callee_args = Vec::with_capacity(r_args.len().saturating_sub(1)); + let mut callee_arg_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); + if method_form { + callee_args.push(r_args[1]); + callee_arg_concretes.push(arg_concretes[1]); + } + callee_args.extend_from_slice(&r_args[2..]); + callee_arg_concretes.extend_from_slice(&arg_concretes[2..]); + (callee_args, callee_arg_concretes) + }; try_walker_inline_resolved_user_call( ctx, op, From d9ac64ebc7f50d0c9858dfda5cd175d5e4238568 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 25 Jul 2026 11:48:03 +0900 Subject: [PATCH 2/4] jit: admit deferred-call callees into FOR_ITER-body inlining `fbw_callee_body_side_effect_free` rejected a callee body at its first `LOAD_GLOBAL` or `box_int` residual. Those two, with `load_const`, carry `CanRaise` only to satisfy the `_OS_CANRAISE` invariant (effectinfo.rs); each is a read or a fresh allocation and commits nothing to the live heap, so the `check_is_elidable() || LoopInvariant` proxy mis-rejected them. Rename the predicate to `fbw_callee_body_replay_safety` and return `CalleeReplaySafety::{Clean, DeferredCall, Dirty}`. The three read/allocate helpers join the provably-side-effect-free set. A `CallFn` / `CallKw` residual no longer forces `Dirty`: its callee is a runtime value, so the body reports `DeferredCall` and the decision moves to the call itself. `fbw_abort_nested_unjournaled_residual` backs that deferral: while a deferred-admitted sub-walk is active it aborts before executing any residual that is not provably side-effect-free, and records the outermost deferred callee in a deny set the gate then consults, so the abort costs one attempt per callee rather than one per trace attempt. The static scan clears every direct heap write and the backstop clears every impure residual, so a deferred sub-walk is write-free wherever it can abort or deopt, which is what both the walk-abort replay and the caller-boundary deopt re-execute. `for i in range(N): total += helper(i)` with `helper(i)` calling `add(i, 1, 2)` goes from 3.67s to 0.11s at N=2M, matching the `while`-loop form; `synth/call_kw_star` from 1.15s to 0.18s. Assisted-by: Claude --- .../src/jitcode_dispatch/fbw_state.rs | 148 ++++++++++++++++-- .../src/jitcode_dispatch/inline_call.rs | 33 +++- 2 files changed, 158 insertions(+), 23 deletions(-) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index 53c57502220..8425b34b663 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -972,6 +972,61 @@ thread_local! { /// trace executes that residual once on later iterations, so the generic /// nested-replay decline does not apply to this resolved descriptor path. pub(crate) static EXCEPTION_STRING_INLINE_ACTIVE: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// Code keys of the callees a FOR_ITER body admitted under + /// [`CalleeReplaySafety::DeferredCall`], outermost first. Non-empty for + /// the lifetime of such a sub-walk ([`ForiterDeferredInlineGuard`]), which + /// is what arms the deferred-call arm of + /// [`fbw_abort_nested_unjournaled_residual`]. + static FBW_FORITER_DEFERRED_INLINE: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + /// Callee code keys whose deferred body reached a CALL residual the lever + /// could not inline. The gate declines them up front from then on, so the + /// backstop abort costs one attempt per callee instead of storming. + static FBW_FORITER_DEFERRED_DENY: std::cell::RefCell> = + std::cell::RefCell::new(std::collections::HashSet::new()); +} + +/// Marks the sub-walk of a callee admitted into a FOR_ITER body under +/// [`CalleeReplaySafety::DeferredCall`] for its whole lifetime, so a nested +/// residual the lever could not inline can recognise the admission it breaks +/// (and the callee to deny) rather than executing. +pub(crate) struct ForiterDeferredInlineGuard(bool); + +impl ForiterDeferredInlineGuard { + pub(crate) fn enter(callee_code_key: usize, deferred: bool) -> Self { + if deferred { + FBW_FORITER_DEFERRED_INLINE.with(|c| c.borrow_mut().push(callee_code_key)); + } + ForiterDeferredInlineGuard(deferred) + } +} + +impl Drop for ForiterDeferredInlineGuard { + fn drop(&mut self) { + if self.0 { + FBW_FORITER_DEFERRED_INLINE.with(|c| { + c.borrow_mut().pop(); + }); + } + } +} + +/// The outermost callee the active sub-walk was admitted for under +/// [`CalleeReplaySafety::DeferredCall`], or `None` outside such a sub-walk. +/// Declining that callee suppresses the whole nest: a body that calls another +/// is itself `DeferredCall`, so no admitted caller sits above it. +fn fbw_foriter_deferred_inline_outermost() -> Option { + FBW_FORITER_DEFERRED_INLINE.with(|c| c.borrow().first().copied()) +} + +pub(crate) fn fbw_foriter_deferred_call_denied(callee_code_key: usize) -> bool { + FBW_FORITER_DEFERRED_DENY.with(|c| c.borrow().contains(&callee_code_key)) +} + +fn fbw_foriter_deny_deferred_call(callee_code_key: usize) { + FBW_FORITER_DEFERRED_DENY.with(|c| { + c.borrow_mut().insert(callee_code_key); + }); } /// Whether the active inline sub-walk is one of the hazard classes the blanket @@ -1032,6 +1087,15 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( // nested-decline guard, which is for FOREIGN unjournaled residuals. let in_selfrec_fold = SELFREC_CA_FOLD_ACTIVE.with(|c| c.get()); let in_exception_string_inline = EXCEPTION_STRING_INLINE_ACTIVE.with(|c| c.get()); + // A FOR_ITER-body inline admitted under `CalleeReplaySafety::DeferredCall` + // stands on the promise that the sub-walk commits nothing: the static scan + // cleared every direct heap write, leaving only Python-level CALL residuals + // whose callee the lever resolves here. One that did not inline breaks the + // promise, so abort BEFORE it executes — every op the sub-walk has run so + // far is write-free, so the resume re-runs the body benignly. Denying the + // admitted callee makes the next attempt decline it statically, so this + // costs one abort per callee rather than an abort per trace attempt. + let foriter_deferred_inline = fbw_foriter_deferred_inline_outermost(); // Narrowed decline: the general depth-≥2 nested // residual inline is sound now that the portal-runner ABI is correct — a // straight-line mutating callee inlines bit-exact. Only two callee shapes @@ -1048,8 +1112,11 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( if !in_selfrec_fold && !in_exception_string_inline && !ctx.session.borrow().framestack.is_empty() - && fbw_inline_callee_hazardous(ctx) + && (foriter_deferred_inline.is_some() || fbw_inline_callee_hazardous(ctx)) { + if let Some(callee_code_key) = foriter_deferred_inline { + fbw_foriter_deny_deferred_call(callee_code_key); + } let (outer_resume, stack_overrides) = { let session = ctx.session.borrow(); match session.framestack.first().and_then(|f| f.parent.as_ref()) { @@ -1276,11 +1343,31 @@ pub(crate) fn fbw_abort_resume_py_pc( Some(python_pc_for_jitcode_pc(&jc.payload.metadata, abort_jit_pc) as usize) } +/// Replay safety of one inline candidate's body inside a FOR_ITER body, as +/// judged by [`fbw_callee_body_replay_safety`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum CalleeReplaySafety { + /// No op in the body can commit a live-heap effect. + Clean, + /// Clean apart from Python-level CALL residuals, whose callee is resolved + /// only at walk time. + DeferredCall, + /// Carries a live-heap effect a replay would double. + Dirty, +} + /// Whether an inline callee can be replayed from its caller's CALL boundary /// without duplicating a live-heap effect. The inline sub-walk's deopt /// snapshot does not yet carry its own callee frame, so this is deliberately -/// stricter than ordinary inlining: unknown calls and every live-heap write -/// decline up front. +/// stricter than ordinary inlining: every live-heap write declines up front. +/// +/// A Python-level CALL residual is the one shape this static scan cannot +/// settle: its callee is a runtime value, so whether the sub-walk inlines it +/// (leaving nothing to replay) or executes it (which may write) is known only +/// at the call. Those bodies report [`CalleeReplaySafety::DeferredCall`] and +/// the lever decides at the call — see +/// [`fbw_abort_nested_unjournaled_residual`], which aborts before executing a +/// residual that did not inline. Every other unproven residual is `Dirty`. /// /// A `new_with_vtable/d>r` result is fresh within this body. Its /// initialization write is benign only when the target field is immutable; @@ -1288,29 +1375,30 @@ pub(crate) fn fbw_abort_resume_py_pc( /// pass through `ref_copy`, but every other Ref-producing instruction clears /// it, so a later `setfield_gc` cannot accidentally be classified as an /// initialization of an earlier allocation. -pub(crate) fn fbw_callee_body_side_effect_free( +pub(crate) fn fbw_callee_body_replay_safety( body_code: &[u8], args_all_numeric: bool, num_regs_i: usize, constants_i: &[i64], callee_descr_refs: &[DescrRef], -) -> bool { +) -> CalleeReplaySafety { let mut fresh_ref_regs = [false; u8::MAX as usize + 1]; + let mut deferred_call = false; let mut pc = 0usize; while pc < body_code.len() { let Some(d) = crate::jitcode_runtime::decode_op_at(body_code, pc) else { - return false; + return CalleeReplaySafety::Dirty; }; if d.opname.starts_with("residual_call") { let Some(descr_index) = residual_call_descr_index_in_body(body_code, &d) else { - return false; + return CalleeReplaySafety::Dirty; }; let Some(call_descr) = callee_descr_refs .get(descr_index) .and_then(|descr| descr.as_call_descr()) else { - return false; + return CalleeReplaySafety::Dirty; }; let ei = call_descr.get_extra_info(); // `ForIterNext` is deliberately not accepted here: it advances the @@ -1319,8 +1407,19 @@ pub(crate) fn fbw_callee_body_side_effect_free( // double-consume. A FOR_ITER-bearing body is declined anyway — its // mandatory `GET_ITER` (`MayForce`) predecessor fails this scan // first — so this only removes a latent landmine, not live inlines. - let provably_side_effect_free = - ei.check_is_elidable() || ei.extraeffect == majit_ir::ExtraEffect::LoopInvariant; + // `load_const` / `load_global` / `box_int` are tagged `CanRaise` + // only to keep the `_OS_CANRAISE` invariant (effectinfo.rs); each + // is a read or a fresh allocation, so re-running one commits + // nothing to the live heap. + let replay_safe_read = matches!( + ei.pyre_helper, + majit_ir::PyreHelperKind::LoadConst + | majit_ir::PyreHelperKind::LoadGlobal + | majit_ir::PyreHelperKind::BoxInt + ); + let provably_side_effect_free = replay_safe_read + || ei.check_is_elidable() + || ei.extraeffect == majit_ir::ExtraEffect::LoopInvariant; if !provably_side_effect_free && !residual_call_is_specialized_plain_int_add( body_code, @@ -1331,13 +1430,26 @@ pub(crate) fn fbw_callee_body_side_effect_free( callee_descr_refs, ) { - return false; + // A Python-level CALL is the one shape this scan cannot + // settle: the inline lever binds its callee only at the call, + // so whether it leaves a residual behind — and what that + // residual writes — is not a property of this body. Defer it; + // the backstop aborts before executing one that did not + // inline. + if matches!( + ei.pyre_helper, + majit_ir::PyreHelperKind::CallFn | majit_ir::PyreHelperKind::CallKw + ) { + deferred_call = true; + } else { + return CalleeReplaySafety::Dirty; + } } } else if d.opname.starts_with("setfield_gc") { // Canonical setfield shapes are `rd`: the target ref is // operand 0 and the field descr is operand 2. let Some(&target_reg) = body_code.get(d.pc + 1) else { - return false; + return CalleeReplaySafety::Dirty; }; let descr_index = decode_descr_index(body_code, &d, 2); let immutable_field = callee_descr_refs @@ -1345,7 +1457,7 @@ pub(crate) fn fbw_callee_body_side_effect_free( .and_then(|descr| descr.as_field_descr()) .is_some_and(|field| field.is_immutable()); if !fresh_ref_regs[target_reg as usize] || !immutable_field { - return false; + return CalleeReplaySafety::Dirty; } } else if d.opname.starts_with("setarrayitem_gc") || d.opname.starts_with("setinteriorfield_gc") @@ -1356,13 +1468,13 @@ pub(crate) fn fbw_callee_body_side_effect_free( { // Array/interior/raw stores and non-residual call forms cannot be // proven replay-safe from this single callee body. - return false; + return CalleeReplaySafety::Dirty; } // The result byte is always the final operand for `>r` forms. if d.argcodes.ends_with(">r") { let Some(&dst) = body_code.get(d.next_pc.saturating_sub(1)) else { - return false; + return CalleeReplaySafety::Dirty; }; fresh_ref_regs[dst as usize] = d.key == "new_with_vtable/d>r" || (d.key == "ref_copy/r>r" @@ -1372,7 +1484,11 @@ pub(crate) fn fbw_callee_body_side_effect_free( } pc = d.next_pc; } - true + if deferred_call { + CalleeReplaySafety::DeferredCall + } else { + CalleeReplaySafety::Clean + } } pub(crate) fn fbw_callee_body_has_binary_op_residual( diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index b1475fb294b..4c88355e3fb 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -1594,18 +1594,35 @@ pub(crate) fn try_walker_inline_resolved_user_call( // An inline sub-walk inside a FOR_ITER body resumes a guard at the // caller's CALL boundary, so deopt re-executes the whole callee. Replaying // a live-heap mutation would double it; the nested-residual decline catches - // that only after an abort storm. A side-effect-free callee replays - // benignly, so admit it. - if fbw_foriter_inflight_active() - && !fbw_callee_body_side_effect_free( + // that only after an abort storm. A callee whose body commits nothing + // replays benignly, so admit it. + // + // A body whose only unproven ops are Python-level CALL residuals is + // admitted too: this same gate re-runs for each callee the lever resolves + // below it, and one it cannot inline aborts before executing + // (`fbw_abort_nested_unjournaled_residual`) and denies this callee, so a + // deferred body commits nothing either. Without that the whole nest + // declines — `helper(i)` calling `add(i, 1, 2)` residualizes both calls + // per iteration, though each body on its own is pure arithmetic. + let mut foriter_deferred_admit = false; + if fbw_foriter_inflight_active() { + let admit = match fbw_callee_body_replay_safety( body.code, args_all_numeric, body.num_regs_i, body.constants_i, callee_descr_refs, - ) - { - return Ok(None); + ) { + CalleeReplaySafety::Clean => true, + CalleeReplaySafety::DeferredCall => { + foriter_deferred_admit = !fbw_foriter_deferred_call_denied(callee_code_key); + foriter_deferred_admit + } + CalleeReplaySafety::Dirty => false, + }; + if !admit { + return Ok(None); + } } if method_form && !allow_method_load_attr @@ -2346,6 +2363,8 @@ pub(crate) fn try_walker_inline_resolved_user_call( // Track this callee for the lifetime of the sub-walk so nested // self-calls see the correct recursion depth. let _inline_frame = InlineFrameGuard::enter(ctx.session, callee_code_key, parent_frame); + let _foriter_deferred = + ForiterDeferredInlineGuard::enter(callee_code_key, foriter_deferred_admit); if let Some(frame) = ActiveResumeFrame::current(ctx.session, ctx.fbw_mode.snapshot_sym) { if frame.body_matches(&body) { seed_callee_vstack_mirror(&mut sub_wc, &frame); From 732f1193bd445dfe2b2caf0c6e561a7f344d829d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 25 Jul 2026 13:38:16 +0900 Subject: [PATCH 3/4] jit: virtualize array-backed BUILD_TUPLE and inline CALL_FUNCTION_EX star calls `try_walker_specialize_newtuple_object` re-emits the canonical `W_TupleObject` shape (`new_with_vtable` + `w_class` / `wrappeditems` `setfield_gc` over a fresh items block) for the BUILD_TUPLE arities `makespecialisedtuple2` does not claim, reading the elements out of the array heap-cache so the `new_array_clear` build keeps no consumer. Arity 2 and the empty tuple are declined. Dispatched from `dispatch_residual_call_iRd_kind` after the existing `spec_ii` fold. `fbw_callee_body_replay_safety` tracks `new_array*` results as fresh and accepts a `setarrayitem_gc` into a fresh array as an initialization, adds `NewtupleFromArray` / `NewlistFromArray` to the replay-safe read/alloc set, and adds `CallFunctionEx` to the deferred-call set. `fbw_unpack_call_function_ex_args` binds a `f(*args)` star tuple to positional arguments by reading the element boxes from the heap cache, so the fold applies when the tuple is virtual at the call. It declines a `**` merge, an arity that is not the callee's `co_argcount`, the method form, and a tuple with no cached backing block. synth/call_function_ex_star: 0.89s -> 0.14s user, loops_aborted 0. check.py dynasm 308/308 + cranelift 308/308. Assisted-by: Claude --- .../src/jitcode_dispatch/fbw_state.rs | 52 +++++++--- .../src/jitcode_dispatch/inline_call.rs | 75 ++++++++++++++- .../src/jitcode_dispatch/residual_call.rs | 14 +++ .../src/jitcode_dispatch/specialize.rs | 94 +++++++++++++++++++ 4 files changed, 222 insertions(+), 13 deletions(-) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index 8425b34b663..8eeaa57e851 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -1369,12 +1369,16 @@ pub(crate) enum CalleeReplaySafety { /// [`fbw_abort_nested_unjournaled_residual`], which aborts before executing a /// residual that did not inline. Every other unproven residual is `Dirty`. /// -/// A `new_with_vtable/d>r` result is fresh within this body. Its -/// initialization write is benign only when the target field is immutable; -/// `wrapint` is the important instance (`W_IntObject.intval`). Freshness may -/// pass through `ref_copy`, but every other Ref-producing instruction clears -/// it, so a later `setfield_gc` cannot accidentally be classified as an -/// initialization of an earlier allocation. +/// A `new_with_vtable/d>r` or `new_array*` result is fresh within this body. +/// A `setfield_gc` initialization write into one is benign only when the +/// target field is immutable (`wrapint` is the important instance, +/// `W_IntObject.intval`); a `setarrayitem_gc` into a fresh array is benign +/// outright, since replay writes the replay's own array (`BUILD_TUPLE` / +/// `BUILD_LIST` fill their backing block this way). Freshness may pass +/// through `ref_copy`, but every other Ref-producing instruction clears it, +/// so a later store cannot accidentally be classified as an initialization of +/// an earlier allocation, and every branch drops the whole set — past control +/// flow the scan cannot say which allocation a register holds. pub(crate) fn fbw_callee_body_replay_safety( body_code: &[u8], args_all_numeric: bool, @@ -1410,12 +1414,16 @@ pub(crate) fn fbw_callee_body_replay_safety( // `load_const` / `load_global` / `box_int` are tagged `CanRaise` // only to keep the `_OS_CANRAISE` invariant (effectinfo.rs); each // is a read or a fresh allocation, so re-running one commits - // nothing to the live heap. + // nothing to the live heap. The BUILD_TUPLE / BUILD_LIST array + // consumers are the same shape one level up: they read a + // freshly-built backing array and return a brand-new container. let replay_safe_read = matches!( ei.pyre_helper, majit_ir::PyreHelperKind::LoadConst | majit_ir::PyreHelperKind::LoadGlobal | majit_ir::PyreHelperKind::BoxInt + | majit_ir::PyreHelperKind::NewtupleFromArray + | majit_ir::PyreHelperKind::NewlistFromArray ); let provably_side_effect_free = replay_safe_read || ei.check_is_elidable() @@ -1438,7 +1446,9 @@ pub(crate) fn fbw_callee_body_replay_safety( // inline. if matches!( ei.pyre_helper, - majit_ir::PyreHelperKind::CallFn | majit_ir::PyreHelperKind::CallKw + majit_ir::PyreHelperKind::CallFn + | majit_ir::PyreHelperKind::CallKw + | majit_ir::PyreHelperKind::CallFunctionEx ) { deferred_call = true; } else { @@ -1459,16 +1469,33 @@ pub(crate) fn fbw_callee_body_replay_safety( if !fresh_ref_regs[target_reg as usize] || !immutable_field { return CalleeReplaySafety::Dirty; } - } else if d.opname.starts_with("setarrayitem_gc") - || d.opname.starts_with("setinteriorfield_gc") + } else if d.opname.starts_with("setarrayitem_gc") { + // The dual of the `setfield_gc` rule: a store into an array this + // body just allocated is an initialization, not a live-heap write, + // so replaying it writes the replay's own fresh array. The + // canonical shapes put the array register in operand 0 (`r…`); the + // `iiid` raw-address form carries no array register to prove fresh. + let target_fresh = d.argcodes.starts_with('r') + && body_code + .get(d.pc + 1) + .is_some_and(|reg| fresh_ref_regs[*reg as usize]); + if !target_fresh { + return CalleeReplaySafety::Dirty; + } + } else if d.opname.starts_with("setinteriorfield_gc") || d.opname.starts_with("raw_store") || d.opname.starts_with("cond_call") || d.opname.starts_with("call_assembler") || d.opname.starts_with("inline_call") { - // Array/interior/raw stores and non-residual call forms cannot be - // proven replay-safe from this single callee body. + // Interior/raw stores and non-residual call forms cannot be proven + // replay-safe from this single callee body. return CalleeReplaySafety::Dirty; + } else if d.opname.starts_with("goto") || d.opname.starts_with("label") { + // Freshness is a straight-line property. Past a branch or a join + // the scan cannot say which allocation a register holds, so drop + // every freshness claim rather than carry one across control flow. + fresh_ref_regs = [false; u8::MAX as usize + 1]; } // The result byte is always the final operand for `>r` forms. @@ -1477,6 +1504,7 @@ pub(crate) fn fbw_callee_body_replay_safety( return CalleeReplaySafety::Dirty; }; fresh_ref_regs[dst as usize] = d.key == "new_with_vtable/d>r" + || d.opname.starts_with("new_array") || (d.key == "ref_copy/r>r" && body_code .get(d.pc + 1) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 4c88355e3fb..2acd507e8cc 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -1346,6 +1346,65 @@ unsafe fn fbw_reorder_call_kw_args( Some((out_args, out_conc)) } +/// Unpack a `bh_call_function_ex_fn(callable, self_or_null, starargs, +/// kwargs_or_null)` star tuple into the positional argument boxes the inline +/// path seeds from, or `None` to leave the call a residual. +/// +/// The elements are read out of the heap cache rather than off the tuple, so +/// this folds exactly when the star tuple is virtual at the call — the +/// `args = (...)` / `f(*args)` pair the walker just recorded, whose +/// `wrappeditems` block and per-index stores are still cached +/// (`try_walker_specialize_newtuple_object`). A tuple that arrived from +/// anywhere else has no cached block and declines, as does any `**kwargs` +/// merge (the helper's `kwargs_or_null` is then a real mapping) and any arity +/// that is not the callee's exact parameter count. +fn fbw_unpack_call_function_ex_args( + ctx: &mut WalkContext<'_, '_, Sym>, + r_args: &[OpRef], + arg_concretes: &[ConcreteValue], + nparams: usize, +) -> Option<(Vec, Vec)> { + if r_args.len() < 4 || arg_concretes.len() < 4 || nparams == 0 { + return None; + } + // `kwargs_or_null` is the checked `PY_NULL` sentinel for a call with no + // `**` merge; anything else is a real mapping this fold does not bind. + match arg_concretes[3] { + ConcreteValue::Null => {} + ConcreteValue::Ref(kwargs) if kwargs.is_null() || kwargs == pyre_object::PY_NULL => {} + _ => return None, + } + let starargs = r_args[2]; + let items_descr = crate::descr::tuple_wrappeditems_descr(); + let block = ctx + .trace_ctx + .heapcache_getfield_cached(starargs, items_descr.index())?; + // The cached length pins the arity: the callee takes exactly `nparams` + // positional parameters, and a mismatch is a runtime TypeError the inline + // path does not model. + let len_op = ctx.trace_ctx.heap_cache().arraylen(block)?; + match len_op.inline_const_to_value() { + Some(majit_ir::Value::Int(n)) if n as usize == nparams => {} + _ => return None, + } + let array_descr_index = crate::state::pyobject_gcarray_descr().index(); + let mut args = Vec::with_capacity(nparams); + let mut concretes = Vec::with_capacity(nparams); + for index in 0..nparams { + let elem = ctx.trace_ctx.heapcache_getarrayitem( + block, + OpRef::ConstInt(index as i64), + array_descr_index, + )?; + // Take each concrete from the element box itself, not from the tuple, + // so the seeded shadow is the one the symbolic argument carries. + let concrete = walker_concrete_ref_object(ctx, elem)?; + args.push(elem); + concretes.push(ConcreteValue::Ref(concrete)); + } + Some((args, concretes)) +} + pub(crate) fn try_walker_inline_user_call( ctx: &mut WalkContext<'_, '_, Sym>, op: &DecodedOp, @@ -1375,7 +1434,8 @@ pub(crate) fn try_walker_inline_user_call( // residual_call site (inlinable calls get their own inline_call jitcodes); // this restores that invariant for the pyre FBW inline-at-residual lever. let is_call_kw = pyre_helper == majit_ir::PyreHelperKind::CallKw; - if pyre_helper != majit_ir::PyreHelperKind::CallFn && !is_call_kw { + let is_call_function_ex = pyre_helper == majit_ir::PyreHelperKind::CallFunctionEx; + if pyre_helper != majit_ir::PyreHelperKind::CallFn && !is_call_kw && !is_call_function_ex { return Ok(None); } if r_args.is_empty() { @@ -1427,6 +1487,19 @@ pub(crate) fn try_walker_inline_user_call( return Ok(None); }; reordered + } else if is_call_function_ex { + // `f(*args)` unpacks the star tuple into positional arguments, which is + // a trace-time reorder whenever the tuple is the one this trace just + // built: its element boxes are still in the heap cache, so the callee + // seeds from them and the tuple keeps no consumer. + if method_form { + return Ok(None); + } + let Some(unpacked) = fbw_unpack_call_function_ex_args(ctx, r_args, &arg_concretes, nparams) + else { + return Ok(None); + }; + unpacked } else { let mut callee_args = Vec::with_capacity(r_args.len().saturating_sub(1)); let mut callee_arg_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 6155a8ea0b6..2143d5544e7 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -2575,6 +2575,20 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } + // The arities `makespecialisedtuple2` does not claim take the canonical + // array-backed `W_TupleObject` shape instead, so a non-escaping BUILD_TUPLE + // of any width folds away rather than allocating through the opaque + // residual. Reached only after the `spec_ii` fold above declines; falls + // through to the residual for any shape it cannot reproduce (SAFE — never + // declined). + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::NewtupleFromArray + && try_walker_specialize_newtuple_object(ctx, op.pc, &r_args, dst, dst_bank)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + // #171: virtualize a non-escaping BUILD_LIST (`newlist_from_array`) by // decomposing it into the `opimpl_newlist` shape (`pyjitpl.py`) — // `new_with_vtable` + `new_array` + `setarrayitem_gc` + `setfield_gc` — diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 9cc9ffb5bb2..4f97005ca34 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -2197,6 +2197,100 @@ pub(crate) fn try_walker_specialize_newlist( Ok(Some(())) } +/// FBW virtualization of the array-backed BUILD_TUPLE — the arities +/// `makespecialisedtuple2` does not claim. Sibling of +/// [`try_walker_specialize_newtuple`] (arity-2 plain-int `spec_ii`) and +/// [`try_walker_specialize_newlist`], reached only after the `spec_ii` fold +/// declines, so that path stays byte-identical. +/// +/// `lower_tuple_build_hlop_to_insn` lowers BUILD_TUPLE to `new_array_clear` + +/// per-index `setarrayitem_gc` + a `newtuple_from_array` residual. Re-emit the +/// canonical `W_TupleObject` shape walker-native (`new_with_vtable` + +/// `w_class` / `wrappeditems` `setfield_gc` over a fresh items block), reading +/// the elements straight out of the array heap-cache so the array build keeps +/// no consumer and DCEs. A tuple that never escapes then folds away entirely, +/// and one that does escape materializes from the same fields the residual +/// would have written. +/// +/// Arity 2 is `makespecialisedtuple2` territory (`Cls_ii` / `Cls_ff` / +/// `Cls_oo`, `specialisedtupleobject.py`): the runtime never builds an +/// array-backed tuple there, so emitting one would diverge from what the +/// blackhole rebuilds on deopt. Declined here — the `spec_ii` fold owns the +/// int-int case and the residual owns the rest. The empty tuple is declined +/// too (no element to recover a length from). +/// +/// Returns `Ok(Some(()))` when folded; `Ok(None)` falls through to the opaque +/// residual, which stays correct for any shape — a non-const array length or +/// an element without a concrete Ref shadow is not declined, just not folded. +pub(crate) fn try_walker_specialize_newtuple_object( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + r_args: &[OpRef], + dst: usize, + dst_bank: char, +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor || dst_bank != 'r' { + return Ok(None); + } + if r_args.len() != 1 { + return Ok(None); + } + let arr = r_args[0]; + // Const backing-array length (`new_array_clear(Const(len))` seeded + // `heapcache.arraylen`; a cleared array has every slot set, so read the + // length directly rather than probing getarrayitem until a miss). + let len = { + let Some(len_op) = ctx.trace_ctx.heap_cache().arraylen(arr) else { + return Ok(None); + }; + match len_op.inline_const_to_value() { + Some(majit_ir::Value::Int(n)) if n >= 1 => n as usize, + _ => return Ok(None), + } + }; + if len == 2 { + return Ok(None); + } + + // Element boxes the BUILD_TUPLE `setarrayitem_gc` ops stored; a cache miss + // (clobbered array / non-const index) bails to the opaque residual. + let descr_idx = crate::state::pyobject_gcarray_descr().index(); + let mut items: Vec = Vec::with_capacity(len); + for i in 0..len { + let Some(elem) = + ctx.trace_ctx + .heapcache_getarrayitem(arr, OpRef::ConstInt(i as i64), descr_idx) + else { + return Ok(None); + }; + items.push(elem); + } + let mut concretes: Vec = Vec::with_capacity(len); + for &it in &items { + let Some(obj) = walker_concrete_ref_object(ctx, it) else { + return Ok(None); + }; + concretes.push(obj); + } + + // Concrete shadow: a fresh array-backed tuple from the element shadows + // (`w_tuple_new` parity for every arity but 2). A new allocation with no + // heap mutation, safe during the walk like `wrapint`. Built before the + // emit so a failure leaves no orphan ops in the trace. + let result_concrete = pyre_object::w_tuple_new_array_backed(concretes); + if result_concrete.is_null() { + return Ok(None); + } + + let tuple_op = crate::helpers::emit_object_tuple_inline(ctx.trace_ctx, &items); + ctx.trace_ctx.set_opref_concrete( + tuple_op, + majit_ir::Value::Ref(majit_ir::GcRef(result_concrete as usize)), + ); + write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, tuple_op)?; + Ok(Some(())) +} + /// #195 / #73: FBW virtualization of an arity-2 plain-int BUILD_TUPLE. /// `lower_tuple_build_hlop_to_insn` lowers BUILD_TUPLE to `new_array_clear` /// + per-index `setarrayitem_gc` + a `newtuple_from_array` residual From 8ec7b3150ec098215078cadc787735f575b3c2de Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 25 Jul 2026 13:38:48 +0900 Subject: [PATCH 4/4] jit: drop callee-body freshness at branch targets, not at branch sites `fbw_callee_body_replay_safety` cleared its `fresh_ref_regs` set when the linear scan reached a `goto`. A join whose incoming branch sits earlier in the body then kept whatever freshness the fall-through predecessor had: in `r1 = getfield_gc(...); goto L; r1 = new_array; L: setarrayitem_gc r1` the scan reads `r1` as fresh at `L` although one path arrives holding a live-heap reference. Collect every label operand up front (`body_branch_targets`, covering the `goto` family, `catch_exception` and `int_*_jump_if_ovf`) and clear at the target pcs instead. A pc with no incoming branch edge keeps the exact linear state, so a conditional goto's fall-through no longer discards freshness. A label this decode cannot locate reports `Dirty`. No current producer reaches the join: `setarrayitem_gc` is emitted only by the BUILD_TUPLE / BUILD_LIST lowerings, whose target register is defined in the same basic block. Measured neutral on call_kw_hot_loop, call_kw_star, call_ex_kwargs_mapping and call_function_ex_star. Assisted-by: Claude --- .../src/jitcode_dispatch/fbw_state.rs | 54 ++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index 8eeaa57e851..fe4bd352d46 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -1343,6 +1343,44 @@ pub(crate) fn fbw_abort_resume_py_pc( Some(python_pc_for_jitcode_pc(&jc.payload.metadata, abort_jit_pc) as usize) } +/// Every pc in `body_code` that some op can branch to: the `goto` family and +/// `catch_exception` carry their target as the label operand, and +/// `int_*_jump_if_ovf` carries an overflow target ahead of its operands. +/// +/// `None` when an op carries a label this decode cannot locate — a var-list +/// or a pyre payload ahead of the `L` — since a missed target would let a +/// freshness claim survive a join it does not hold across. +fn body_branch_targets(body_code: &[u8]) -> Option> { + let mut targets = std::collections::HashSet::new(); + let mut pc = 0usize; + while pc < body_code.len() { + let d = crate::jitcode_runtime::decode_op_at(body_code, pc)?; + if d.argcodes.contains('L') { + // Operand widths follow `decode_op_at`; only the fixed-width forms + // can precede the label, so anything else gives up. + let mut cursor = d.pc + 1; + let mut target = None; + for operand in d.argcodes.chars() { + match operand { + 'L' => { + target = Some(u16::from_le_bytes([ + *body_code.get(cursor)?, + *body_code.get(cursor + 1)?, + ]) as usize); + break; + } + 'i' | 'c' | 'r' | 'f' => cursor += 1, + 'd' | 'j' => cursor += 2, + _ => break, + } + } + targets.insert(target?); + } + pc = d.next_pc; + } + Some(targets) +} + /// Replay safety of one inline candidate's body inside a FOR_ITER body, as /// judged by [`fbw_callee_body_replay_safety`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -1377,8 +1415,9 @@ pub(crate) enum CalleeReplaySafety { /// `BUILD_LIST` fill their backing block this way). Freshness may pass /// through `ref_copy`, but every other Ref-producing instruction clears it, /// so a later store cannot accidentally be classified as an initialization of -/// an earlier allocation, and every branch drops the whole set — past control -/// flow the scan cannot say which allocation a register holds. +/// an earlier allocation, and every branch target drops the whole set — a +/// register reaching a join can hold whichever allocation the taken path put +/// there, which this straight-line scan cannot name. pub(crate) fn fbw_callee_body_replay_safety( body_code: &[u8], args_all_numeric: bool, @@ -1386,10 +1425,16 @@ pub(crate) fn fbw_callee_body_replay_safety( constants_i: &[i64], callee_descr_refs: &[DescrRef], ) -> CalleeReplaySafety { + let Some(branch_targets) = body_branch_targets(body_code) else { + return CalleeReplaySafety::Dirty; + }; let mut fresh_ref_regs = [false; u8::MAX as usize + 1]; let mut deferred_call = false; let mut pc = 0usize; while pc < body_code.len() { + if branch_targets.contains(&pc) { + fresh_ref_regs = [false; u8::MAX as usize + 1]; + } let Some(d) = crate::jitcode_runtime::decode_op_at(body_code, pc) else { return CalleeReplaySafety::Dirty; }; @@ -1491,11 +1536,6 @@ pub(crate) fn fbw_callee_body_replay_safety( // Interior/raw stores and non-residual call forms cannot be proven // replay-safe from this single callee body. return CalleeReplaySafety::Dirty; - } else if d.opname.starts_with("goto") || d.opname.starts_with("label") { - // Freshness is a straight-line property. Past a branch or a join - // the scan cannot say which allocation a register holds, so drop - // every freshness claim rather than carry one across control flow. - fresh_ref_regs = [false; u8::MAX as usize + 1]; } // The result byte is always the final operand for `>r` forms.