From dbf3c4273dbb1785b2a8ef3151ea6641c0930911 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 16:00:04 +0900 Subject: [PATCH 01/11] jit: exc-edge bridge fixes (forward catch lookup, dynasm exception-cell ops, stale exception seed) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - walker: add find_catch_for_exc_resume — the blackhole handle_exception_in_frame forward case (catch_exception directly after the resume -live-, blackhole.py:396) tried before the backward scan; the exc-edge routing in dispatch_via_miframe now uses it. - dynasm aarch64/x86: implement SaveExcClass (load pos_exception, assembler.py:1817-1818), SaveException (shared emit_store_and_reset_exception, assembler.py:1820-1821) and RestoreException (assembler.py:1845-1850). The previous SaveExcClass/SaveException bodies returned 0 and RestoreException had no emit arm. - walker: seed_standing_exception_for_walk reads BH_LAST_EXC_VALUE before the preseeded-sym early return, so the exception published from the current guard failure overwrites exception state a previous walk left on the persistent sym (_prepare_exception_resumption grabs from this failure's deadframe, pyjitpl.py:3125-3126). A preseeded sym is kept only when no fresh publish exists. All three changes are exercised only with PYRE_EXC_EDGE_BRIDGE set. check.py 293/293 on dynasm and cranelift. Assisted-by: Claude --- .../src/aarch64/assembler.rs | 53 +++++++++++--- .../majit-backend-dynasm/src/x86/assembler.rs | 73 ++++++++++++++++--- .../src/jitcode_dispatch/bridge_subwalk.rs | 2 +- .../src/jitcode_dispatch/mod.rs | 45 +++++++++++- 4 files changed, 145 insertions(+), 28 deletions(-) diff --git a/majit/majit-backend-dynasm/src/aarch64/assembler.rs b/majit/majit-backend-dynasm/src/aarch64/assembler.rs index 0968f8e549d..62a5b6b0b92 100644 --- a/majit/majit-backend-dynasm/src/aarch64/assembler.rs +++ b/majit/majit-backend-dynasm/src/aarch64/assembler.rs @@ -3388,8 +3388,11 @@ impl<'a> AssemblerARM64<'a> { dynasm!(self.mc ; .arch aarch64 ; mov X(r.value), x29); } } - OpCode::SaveException => self.genop_save_exception(op), - OpCode::SaveExcClass => self.genop_save_exc_class(op), + // assembler.py:1820-1821 genop_save_exception IS + // `_store_and_reset_exception(resloc)` — reuse the shared helper. + OpCode::SaveException => self.emit_store_and_reset_exception(result_loc), + OpCode::SaveExcClass => self.genop_save_exc_class(result_loc), + OpCode::RestoreException => self.genop_restore_exception(arglocs), // Guards never reach the non-guard regalloc dispatch — they // are emitted exclusively from `regalloc_perform_guard` via // the `RegAllocOp::PerformWithGuard` arm. @@ -6468,20 +6471,46 @@ impl<'a> AssemblerARM64<'a> { // assembler.py:1817 genop_save_exc_class / genop_save_exception // ================================================================ - /// assembler.py:1817 genop_save_exc_class — stub: returns 0. - fn genop_save_exc_class(&mut self, op: &Op) { - dynasm!(self.mc ; .arch aarch64 ; mov x0, 0); - if !op.pos.get().is_none() { - self.store_rax_to_result(op.pos.get()); + /// assembler.py:1817-1818 genop_save_exc_class: + /// `MOV resloc, [pos_exception]`. The regalloc always assigns the + /// result a register (`consider_no_arg_result`). + fn genop_save_exc_class(&mut self, result_loc: Option<&Loc>) { + self.emit_mov_imm64(16, crate::jit_exc_type_addr() as i64); + match result_loc { + Some(Loc::Reg(dst)) => { + dynasm!(self.mc ; .arch aarch64 ; ldr X(dst.value), [x16]); + } + Some(Loc::Frame(frame)) => { + dynasm!(self.mc ; .arch aarch64 ; ldr x17, [x16]); + self.emit_str_fp(17, frame.ebp_loc.value); + } + _ => {} } } - /// assembler.py:1827 genop_save_exception — stub: returns 0. - fn genop_save_exception(&mut self, op: &Op) { - dynasm!(self.mc ; .arch aarch64 ; mov x0, 0); - if !op.pos.get().is_none() { - self.store_rax_to_result(op.pos.get()); + /// assembler.py:1845-1850 `_restore_exception`: + /// `MOV [pos_exc_value], excvalloc; MOV [pos_exception], exctploc`. + /// arglocs = [class, value]; the regalloc brings both into registers + /// (`consider_restore_exception`). x16/x17 are the scratch pair. + fn genop_restore_exception(&mut self, arglocs: &[Loc]) { + if arglocs.len() < 2 { + return; } + let load_to_x17 = |this: &mut Self, loc: &Loc| match loc { + Loc::Reg(src) => { + let src = src.value; + dynasm!(this.mc ; .arch aarch64 ; mov x17, X(src)); + } + Loc::Frame(frame) => this.emit_ldr_fp(17, frame.ebp_loc.value), + Loc::Immed(imm) => this.emit_mov_imm64(17, imm.value), + _ => {} + }; + load_to_x17(self, &arglocs[1]); // value + self.emit_mov_imm64(16, crate::jit_exc_value_addr() as i64); + dynasm!(self.mc ; .arch aarch64 ; str x17, [x16]); + load_to_x17(self, &arglocs[0]); // class + self.emit_mov_imm64(16, crate::jit_exc_type_addr() as i64); + dynasm!(self.mc ; .arch aarch64 ; str x17, [x16]); } // ================================================================ diff --git a/majit/majit-backend-dynasm/src/x86/assembler.rs b/majit/majit-backend-dynasm/src/x86/assembler.rs index 5894f75eadd..64deed7aac7 100644 --- a/majit/majit-backend-dynasm/src/x86/assembler.rs +++ b/majit/majit-backend-dynasm/src/x86/assembler.rs @@ -4542,8 +4542,11 @@ impl<'a> Assembler386<'a> { dynasm!(self.mc ; .arch x64 ; mov Rq(r.value), rbp); } } - OpCode::SaveException => self.genop_save_exception(op), - OpCode::SaveExcClass => self.genop_save_exc_class(op), + // assembler.py:1820-1821 genop_save_exception IS + // `_store_and_reset_exception(resloc)` — reuse the shared helper. + OpCode::SaveException => self.emit_store_and_reset_exception(result_loc), + OpCode::SaveExcClass => self.genop_save_exc_class(result_loc), + OpCode::RestoreException => self.genop_restore_exception(arglocs), // Guards never reach the non-guard regalloc dispatch — they // are emitted exclusively from `regalloc_perform_guard` via // the `RegAllocOp::PerformWithGuard` arm @@ -8432,20 +8435,66 @@ impl<'a> Assembler386<'a> { // assembler.py:1817 genop_save_exc_class / genop_save_exception // ================================================================ - /// assembler.py:1817 genop_save_exc_class — stub: returns 0. - fn genop_save_exc_class(&mut self, op: &Op) { - dynasm!(self.mc ; .arch x64 ; xor eax, eax); - if !op.pos.get().is_none() { - self.store_rax_to_result(op.pos.get()); + /// assembler.py:1817-1818 genop_save_exc_class: + /// `MOV resloc, [pos_exception]`. The regalloc always assigns the + /// result a register (`consider_no_arg_result`). + fn genop_save_exc_class(&mut self, result_loc: Option<&Loc>) { + let scratch = crate::regloc::X86_64_SCRATCH_REG.value; + let exc_type_addr = crate::jit_exc_type_addr() as i64; + dynasm!(self.mc ; .arch x64 ; mov Rq(scratch), QWORD exc_type_addr); + match result_loc { + Some(Loc::Reg(dst)) => { + dynasm!(self.mc ; .arch x64 ; mov Rq(dst.value), [Rq(scratch)]); + } + Some(Loc::Frame(frame)) => { + let ofs = frame.ebp_loc.value; + dynasm!(self.mc ; .arch x64 + ; mov Rq(scratch), [Rq(scratch)] + ; mov [rbp + ofs], Rq(scratch) + ); + } + _ => {} } } - /// assembler.py:1827 genop_save_exception — stub: returns 0. - fn genop_save_exception(&mut self, op: &Op) { - dynasm!(self.mc ; .arch x64 ; xor eax, eax); - if !op.pos.get().is_none() { - self.store_rax_to_result(op.pos.get()); + /// assembler.py:1845-1850 `_restore_exception`: + /// `MOV [pos_exc_value], excvalloc; MOV [pos_exception], exctploc`. + /// arglocs = [class, value]; the regalloc brings both into registers + /// (`consider_restore_exception`), so only the Reg arm is hot — the + /// non-Reg fallback round-trips through rax with a push/pop save. + fn genop_restore_exception(&mut self, arglocs: &[Loc]) { + if arglocs.len() < 2 { + return; } + let scratch = crate::regloc::X86_64_SCRATCH_REG.value; + let mut store_loc_to = |this: &mut Self, cell_addr: i64, loc: &Loc| { + dynasm!(this.mc ; .arch x64 ; mov Rq(scratch), QWORD cell_addr); + match loc { + Loc::Reg(src) => { + dynasm!(this.mc ; .arch x64 ; mov [Rq(scratch)], Rq(src.value)); + } + Loc::Frame(frame) => { + let ofs = frame.ebp_loc.value; + dynasm!(this.mc ; .arch x64 + ; push rax + ; mov rax, [rbp + ofs] + ; mov [Rq(scratch)], rax + ; pop rax + ); + } + Loc::Immed(imm) => { + dynasm!(this.mc ; .arch x64 + ; push rax + ; mov rax, QWORD imm.value + ; mov [Rq(scratch)], rax + ; pop rax + ); + } + _ => {} + } + }; + store_loc_to(self, crate::jit_exc_value_addr() as i64, &arglocs[1]); + store_loc_to(self, crate::jit_exc_type_addr() as i64, &arglocs[0]); } // ================================================================ diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index 366a6034a0d..418b79339af 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -214,7 +214,7 @@ pub fn dispatch_via_miframe( && !sym.last_exc_box().is_none() && !sym.last_exc_value().is_null(); let exc_edge_catch_target = if exc_edge_precondition { - find_catch_before_resume_live(jitcode_code, position) + find_catch_for_exc_resume(jitcode_code, position) // Only route when the handler rejoins this frame's loop; a handler // that returns out of the frame (called function's `try/except: // return`, compiled as its own function trace) needs cross-frame diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 3fd76caf8f3..13541be3708 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -2379,6 +2379,35 @@ pub fn carrier_exc_resume_enabled() -> bool { *ENABLED.get_or_init(|| std::env::var_os("PYRE_CARRIER_EXC_RESUME").is_some()) } +/// Mirror of `blackhole.rs BlackholeInterpreter::handle_exception_in_frame` +/// for the walker: locate the `catch_exception/L` that owns an exception-guard +/// resume position, forward case first, then the backward scan. +/// +/// Forward case (blackhole.py:396): the `catch_exception` sits directly after +/// the resume `-live-` — explicit `raise` sites, and residual ops whose resume +/// coordinate is their own post-call `-live-` (a mid-Python-op residual such as +/// a binary-op helper resumes at its OWN op's `-live-`, with its catch +/// following it). Backward case: the after-residual-call layout where the +/// catch sits BEHIND the resume `-live-` (`find_catch_before_resume_live`). +/// Returns the handler target (2-byte LE label), or `None` when the raising op +/// sits outside any in-frame try (propagate). +pub(crate) fn find_catch_for_exc_resume(code: &[u8], resume_live_pos: usize) -> Option { + let mut position = resume_live_pos; + if let Some(op) = decode_op_at(code, position) { + if op.key == "live/" { + position = op.next_pc; + } + if let Some(op) = decode_op_at(code, position) { + if op.key == "catch_exception/L" { + let lo = code[position + 1] as usize; + let hi = code[position + 2] as usize; + return Some(lo | (hi << 8)); + } + } + } + find_catch_before_resume_live(code, resume_live_pos) +} + /// Mirror of `blackhole.rs BlackholeInterpreter::find_catch_before_resume_live` /// for the walker. An after-residual-call exception guard resumes at the /// no-exception fallthrough `-live-` (the next opcode after the call); the @@ -3153,13 +3182,19 @@ fn dispatch_switch_id( /// exception state into bridge tracing. Operand-stack values are never scanned /// to infer a standing exception. fn seed_standing_exception_for_walk(sym: &mut Sym, trace_ctx: &mut TraceCtx) { - if !sym.last_exc_box().is_none() { - return; - } if trace_ctx.is_bridge_trace && !trace_ctx.bridge_source_is_exception_guard() { return; } + // `_prepare_exception_resumption` (pyjitpl.py:3125-3126) reads the + // exception off THIS failure's deadframe (`cpu.grab_exc_value`), and every + // bridge compile runs on a fresh `MetaInterp`, so a previous compile's + // `last_exc_value` can never leak into a new bridge trace. Pyre's sym + // persists across walks; the routed publish (`BH_LAST_EXC_VALUE`, set from + // the same grab) is the fresh-failure signal and must OVERWRITE any + // exception state a previous walk left on the sym. A preseeded sym is + // kept only when no fresh signal exists — the multi-frame carrier walk + // re-seeds per frame after the first frame drained the cell. let bh_exc = majit_metainterp::blackhole::BH_LAST_EXC_VALUE.with(|c| c.get()); if bh_exc != 0 { let exc = bh_exc as pyre_object::PyObjectRef; @@ -3174,6 +3209,10 @@ fn seed_standing_exception_for_walk(sym: &mut Sym, trace_ctx: &mut } } + if !sym.last_exc_box().is_none() { + return; + } + let current = pyre_interpreter::eval::get_current_exception(); if !current.is_null() && unsafe { pyre_object::is_exception(current) } { let exc_box = trace_ctx.const_ref(current as i64); From d58f9085b8c4afb91e41e493e777b246a0d4c5b1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 17:15:43 +0900 Subject: [PATCH 02/11] jit: guard the exception flavor at exception-guard bridge entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A loop trace recorded through a raising iteration carries that iteration's GUARD_EXCEPTION(class). No-raise iterations chronically fail it WITHOUT a pending exception, so a bridge is compiled for the no-exception continuation; a second exception class then enters the same bridge WITH a pending exception and the recorded continuation runs on the NULL raised-call result (SIGSEGV in compiled code, both backends, default mode). - walker: an exception-guard bridge walk with no standing exception now records GUARD_NO_EXCEPTION at bridge entry (_prepare_exception_resumption null arm + prepare_resume_from_failure, pyjitpl.py:3152-3171), so the pending-exception flavor deopts to the blackhole at entry. - cranelift: attached-bridge in-code dispatch now also runs for must_save_exception guards, before the exception staging in emit_guard_exit — entering the bridge with the exception cells intact, as dynasm's patched guard jump does (patch_jump_for_descr). The previous host-loop re-entry consumed the exception before invoking the bridge, so the entry flavor guard could not see it. - call_jit: decline bridge compilation from GUARD_NOT_FORCED failures — "Failures of a GUARD_NOT_FORCED are never compiled, but always just blackholed" (ResumeGuardForcedDescr.handle_fail, compile.py:950-953). - bench: add synth/exc_mixed_classes_bridge_flavor covering the two-exception-class shape. check.py 293/293 on dynasm and cranelift. Assisted-by: Claude --- majit/majit-backend-cranelift/src/compiler.rs | 16 +++++++--- .../synth/exc_mixed_classes_bridge_flavor.py | 32 +++++++++++++++++++ .../src/jitcode_dispatch/bridge_subwalk.rs | 14 ++++++++ pyre/pyre-jit/src/call_jit.rs | 8 +++++ 4 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 pyre/bench/synth/exc_mixed_classes_bridge_flavor.py diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index b079673c6f0..090f70e6dce 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -6751,11 +6751,19 @@ fn emit_guard_exit( } } - if info.can_have_bridge && !info.must_save_exception { + if info.can_have_bridge { // RPython/dynasm patched guards enter the bridge before failure - // recovery. Cranelift still has to publish failargs into jf_frame - // because bridge input locations are frame based, but a bridge hit - // should skip the deadframe-only gcmap/write-barrier/descr stores. + // recovery (`patch_jump_for_descr` rewrites the guard branch, so the + // failure-recovery stub never runs on a bridge hit). Cranelift still + // has to publish failargs into jf_frame because bridge input locations + // are frame based, but a bridge hit should skip the deadframe-only + // gcmap/write-barrier/descr stores. must_save_exception guards + // dispatch here too — BEFORE the exception staging below — so the + // pending-exception cells flow into the bridge intact and its entry + // flavor guard (GUARD_NO_EXCEPTION / GUARD_EXCEPTION, + // `prepare_resume_from_failure`) can check them; staging first would + // consume the exception and let the wrong-flavor entry run the + // recorded continuation on a NULL raised-call result. emit_attached_bridge_dispatch( builder, jf_ptr, diff --git a/pyre/bench/synth/exc_mixed_classes_bridge_flavor.py b/pyre/bench/synth/exc_mixed_classes_bridge_flavor.py new file mode 100644 index 00000000000..cc99913e0d4 --- /dev/null +++ b/pyre/bench/synth/exc_mixed_classes_bridge_flavor.py @@ -0,0 +1,32 @@ +# A callee raises TWO different exception classes into a hot try/except loop. +# The loop trace records the raising iteration's GUARD_EXCEPTION(class-A); the +# no-raise iterations chronically fail that guard WITHOUT a pending exception, +# so a bridge is compiled for the no-exception continuation. When class B +# arrives, the same guard fails WITH a pending exception and enters the same +# bridge — the bridge's entry flavor guard (GUARD_NO_EXCEPTION, +# prepare_resume_from_failure) must deopt that entry to the blackhole instead +# of running the recorded continuation on the NULL raised-call result. +N = 60000 + + +def f(i): + if i % 3 == 1: + raise ValueError(i) + if i % 3 == 2: + raise TypeError(i) + return i + + +def run(): + acc = 0 + for i in range(N): + try: + acc += f(i) + except ValueError: + acc += i + except TypeError: + acc += i * 2 + return acc + + +print(run()) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index 418b79339af..616f97d2861 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -398,6 +398,20 @@ pub fn dispatch_via_miframe( vstack_enter_exception_handler(&mut wc, seed.catch_target, seed.exc); seed.catch_target } else { + // `_prepare_exception_resumption` null-exception arm + // (pyjitpl.py:3152-3154) + `prepare_resume_from_failure` + // (pyjitpl.py:3156-3171): every exception-guard bridge re-checks + // its entry flavor. With no pending exception at walk time, + // `clear_exception()` + `handle_possible_exception()` record + // GUARD_NO_EXCEPTION at the bridge start, so the OTHER failure + // flavor — a pending exception whose class the source guard's + // expected class does not match — deopts to the blackhole at + // bridge entry instead of running the recorded no-exception + // continuation on a NULL raised-call result. + if wc.trace_ctx.is_bridge_trace && wc.trace_ctx.bridge_source_is_exception_guard() { + wc.trace_ctx.record_guard(OpCode::GuardNoException, &[], 0); + walker_capture_snapshot_for_last_guard(&mut wc, position)?; + } seed_vstack_mirror(&mut wc, sym, position); position }; diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index a6253488e8d..89b31ae1395 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -2780,6 +2780,14 @@ pub fn trace_and_compile_from_bridge( use crate::eval::build_jit_state; use crate::jit::state::PyreEnv; + // compile.py:950-953 `ResumeGuardForcedDescr.handle_fail`: "Failures of + // a GUARD_NOT_FORCED are never compiled, but always just blackholed." + // A bridge walked from one force flavor (pure force, live call result) + // would also be entered for forced-and-raised failures whose call result + // is NULL — its result-class guard then dereferences NULL. + if descr_arc.is_guard_forced() { + return BridgeResolution::ResumeBlackhole; + } let Some((green_key, trace_id, fail_index)) = bridge_source_identity_from_descr(descr_arc) else { // compile.py:725-729 `_trace_and_compile_from_bridge` raises From 797adaeed6828c5ae4b92ea0673884772190bacd Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 21:27:16 +0900 Subject: [PATCH 03/11] jit: carry the bridge-entry flavor guard's resume coordinate verbatim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The routed/null bridge-entry flavor-guard captures fed the walk-entry position — already a post-call resume coordinate — through the after-residual capture path, whose op-START-keyed twins advanced it a second time, onto the physically-following except-handler block. The entry guard's own bridge then resumed inside the handler, and its other-flavor decode failed the exc-edge catch lookup (ExcEdgeCrossFrameReturnUnsupported retry loop). Add GuardCaptureScope::carried_resume_jit_pc: the entry captures carry position verbatim as the guard's resume word, take the resume py from the forward twin at that word, and skip the op-START-keyed depth twins (they read the key opcode's depth, over-publishing valuestackdepth so the resume read garbage slots as Refs). Assisted-by: Claude --- .../src/jitcode_dispatch/bridge_subwalk.rs | 34 +++++++++++++++++-- .../src/jitcode_dispatch/mod.rs | 13 +++++++ .../src/jitcode_dispatch/resume_snapshot.rs | 29 +++++++++++++++- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index 616f97d2861..d5e4ae7b23e 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -359,7 +359,26 @@ pub fn dispatch_via_miframe( let exc_class_const = wc.trace_ctx.const_int(exc_edge_class); wc.trace_ctx .record_guard(OpCode::GuardException, &[exc_class_const], 0); - walker_capture_snapshot_for_last_guard(&mut wc, position)?; + // `handle_possible_exception` captures resume data at the MIFrame's + // CURRENT pc — already past the residual call (`pyjitpl.py:2610 + // capture_resumedata`, default `resumepc`). `position` here IS + // that post-call resume coordinate (decoded from the failing + // guard), so capture WITHOUT the after-residual advance and carry + // `position` verbatim (`GuardCaptureScope::carried_resume_jit_pc`): + // the twin lookups compensate CALL-START keys and would advance an + // already-advanced coordinate a second time — on this shape onto + // the physically-following `except` handler block, so the entry + // guard's own bridge would resume INSIDE the handler (every + // no-raise iteration then runs the handler body). + walker_capture_snapshot_for_last_guard_impl( + &mut wc, + position, + false, + GuardCaptureScope { + carried_resume_jit_pc: Some(position), + ..Default::default() + }, + )?; // `execute_ll_raised` parity: the standing exception the handler // reads (`last_exc_value/>r`) is the SAVE_EXCEPTION box — the // runtime-restored value, NOT a baked constant — so a value-using @@ -410,7 +429,18 @@ pub fn dispatch_via_miframe( // continuation on a NULL raised-call result. if wc.trace_ctx.is_bridge_trace && wc.trace_ctx.bridge_source_is_exception_guard() { wc.trace_ctx.record_guard(OpCode::GuardNoException, &[], 0); - walker_capture_snapshot_for_last_guard(&mut wc, position)?; + // `position` is already the post-call resume coordinate — + // capture without the after-residual advance and carry it + // verbatim (see the routed arm above). + walker_capture_snapshot_for_last_guard_impl( + &mut wc, + position, + false, + GuardCaptureScope { + carried_resume_jit_pc: Some(position), + ..Default::default() + }, + )?; } seed_vstack_mirror(&mut wc, sym, position); position diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 13541be3708..b0932e82db7 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -4304,6 +4304,19 @@ pub(crate) struct GuardCaptureScope<'a> { /// residuals fall back to the fallthrough resume even when this is set. pub residual_call_catch_resume: bool, + /// Bridge-entry flavor guard (`_prepare_exception_resumption`, + /// pyjitpl.py:3125-3173): the walk-entry `position` IS the resume + /// coordinate — the failing source guard's own carried word, already a + /// decodable `-live-` startpoint the runtime just resolved. Carry it + /// verbatim and key the resume py off its forward twin. The default + /// twin lookups compensate op-START keys (the after-residual advance, + /// the block-head re-key) and would move an already-advanced coordinate + /// a second time — on a call inside a try-block that lands on the + /// physically-following `except` handler block, so the entry guard's + /// own bridge resumes INSIDE the handler. None outside the bridge-entry + /// captures. + pub carried_resume_jit_pc: Option, + /// The branch guard's own jitcode `op.pc` for a kept-stack branch guard /// (#124). The snapshot helper is invoked with the *resume* coordinate /// (`other_target`, the not-taken arm), so the guard's own coordinate is diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index 4ab323cd9ac..9f71abe1cd2 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -353,6 +353,16 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( if !jc.payload.code_ptr.is_null() { let code = &*jc.payload.code_ptr; py = skip_python_trivia_forward(code, py as usize) as u32; + // Entry-resume capture: the carried coordinate is the + // authority; take its forward-py twin (the same pairing + // the decoder resolves) instead of the inversion above, + // so the liveness/entry windows key exactly where the + // decode will. + if let Some(carried) = scope.carried_resume_jit_pc { + if let Some(fwd) = jc.payload.forward_py_pc_for_jitcode_pc(carried) { + py = fwd; + } + } // after_residual_call=True (`pyjitpl.py`): the // may-force call already executed in compiled code and // consumed its Python stack operands. Resume at the NEXT @@ -420,6 +430,17 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( let jc = &*sym.jitcode(); if loop_close_overshoot { None + } else if scope.carried_resume_jit_pc.is_some() { + // Entry-resume capture: the depth twins are keyed for + // op-START coordinates; keying them with an + // already-advanced resume coordinate reads the depth of + // the coordinate's KEY opcode (the call, with its + // operands still on the stack) — an over-count whose + // published `valuestackdepth` makes the resume read + // garbage slots as Refs. Fall back to the raw static + // liveness at the resume py (the forward twin above), + // exactly what the decode derives. + None } else if after_residual_call { jc.payload .after_residual_marker_for_jitcode_pc(op_pc) @@ -703,7 +724,13 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // the entry coordinate, and for a non-branch guard // `op_pc != marker` // — the two windows diverge and the decoded box layout mismatches. - let guard_jitcode_pc: i32 = if let Some(guard_jc_pc) = scope.branch_guard_jitcode_pc { + let guard_jitcode_pc: i32 = if let Some(carried) = scope.carried_resume_jit_pc { + // Bridge-entry flavor guard: re-carry the source guard's own + // resume word verbatim (see `GuardCaptureScope`). It is a + // decodable `-live-` startpoint by construction — the runtime + // resolved it to reach this walk. + carried as i32 + } else if let Some(guard_jc_pc) = scope.branch_guard_jitcode_pc { // The kept-stack branch guard's own `op.pc` (walker // `MIFrame.pc`) — the ONE carried word not sourced from the // resume-translation. From 5d0b22f453897de914263ed230737808fa774640 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 21:28:30 +0900 Subject: [PATCH 04/11] jit: clear the standing-exception seed on a no-exception bridge failure seed_standing_exception_for_walk kept a preseeded sym exception when the published cell was empty. For an exception-guard bridge the publish is the deadframe-grab authority, so an empty cell now clears the seed (_prepare_exception_resumption null arm, pyjitpl.py:3152-3154). Previously a no-exception failure of an exception guard walked a stale exception's handler as the no-exception continuation, and the per-flavor bridge chain recompiled the same handler indefinitely instead of converging. Assisted-by: Claude --- pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index b0932e82db7..fe905c58e98 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -3209,6 +3209,23 @@ fn seed_standing_exception_for_walk(sym: &mut Sym, trace_ctx: &mut } } + if trace_ctx.is_bridge_trace && trace_ctx.bridge_source_is_exception_guard() { + // Null arm (`pyjitpl.py:3152-3154`): the deadframe published NO + // exception, and for an exception-guard bridge that publish is the + // sole authority — `clear_exception()` on the fresh MetaInterp. + // Keeping a preseeded sym here would walk the exception-flavor + // continuation for a no-exception failure: a loop whose exception + // guard was recorded through a raising iteration then compiles the + // HANDLER as its no-exception bridge, and every non-raising + // iteration runs the handler body. + sym.set_current_exc_value(pyre_object::PY_NULL); + sym.set_current_exc_box(OpRef::NONE); + sym.set_last_exc_value(pyre_object::PY_NULL); + sym.set_last_exc_box(OpRef::NONE); + sym.set_class_of_last_exc_is_const(false); + return; + } + if !sym.last_exc_box().is_none() { return; } From e4f431f8fbddd6d9b5117b36565f847e49b02825 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 23 Jul 2026 23:11:07 +0900 Subject: [PATCH 05/11] gc: write-barrier JIT-side virtualizable frame stores and exc child walks The guard-failure vable sync (write_boxes_to_heap), the walk-end escape flushes, and the MidBody abort commit stored decoded/boxed refs into a frame's locals_cells_stack_w raw. The values can be nursery-young while the frame/array are old-gen and the virtualizable runs detached from the walked frame chain, so no minor re-traced the items; a traceback-reachable frame then fed the stale nursery address to major marking, panicking in incremental_mark_step (invalid type_id; reproducible with MAJIT_GC_STRESS=1 PYRE_EXC_EDGE_BRIDGE=1 on a reraise loop). - majit-metainterp write_field / write_array_item: arm the object/array in the remembered set after every Ref store (virtualizable.py:101-113 write_boxes stores run under the translated write barrier upstream). - pyre-jit-trace flush/commit paths and execute_assembler entry: re-arm the frame + array via frame_array_write_barrier. - Exception root walkers (walk_jit_exc_value, walk_active_sym_exc_roots, PyError::walk_gc_refs): forward the non-moving carrier's raw child slots so young tracebacks/args parked across a minor stay valid; add the missing BH_LAST_EXC_VALUE walker. Verified: stress battery clean on both backends; adversarial exc battery matches CPython flag-on and flag-off; check.py 298/298 dynasm and cranelift. Assisted-by: Claude --- majit/majit-metainterp/src/virtualizable.rs | 20 ++++++++ pyre/pyre-interpreter/src/error.rs | 5 ++ pyre/pyre-jit-trace/src/state.rs | 27 +++++++++++ pyre/pyre-jit-trace/src/trace.rs | 41 ++++++++++++---- pyre/pyre-jit/src/eval.rs | 54 +++++++++++++++++++-- 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/majit/majit-metainterp/src/virtualizable.rs b/majit/majit-metainterp/src/virtualizable.rs index 6d2c81f87ca..864fd974092 100644 --- a/majit/majit-metainterp/src/virtualizable.rs +++ b/majit/majit-metainterp/src/virtualizable.rs @@ -1064,6 +1064,14 @@ impl VirtualizableInfo { Type::Ref => { let ptr = obj_ptr.add(field.offset) as *mut usize; *ptr = value as usize; + // The ref may be nursery-young while the virtualizable is + // old-gen and runs detached from the walked frame chain; + // upstream's write_boxes stores run under the translated + // write barrier (virtualizable.py:101-113), so arm the + // object in the remembered set here. + if majit_gc::gc_owns_object(obj_ptr as usize) { + majit_gc::gc_write_barrier(majit_ir::GcRef(obj_ptr as usize)); + } } // Word-sized `Signed` field (`isize`/`usize`): 4 bytes on // wasm32. Writing 8 bytes would clobber the adjacent field. @@ -1168,6 +1176,18 @@ impl VirtualizableInfo { Type::Ref => { let ptr = array_ptr.add(item_offset) as *mut usize; *ptr = value as usize; + // The ref may be nursery-young while the array/frame are + // old-gen and run detached from the walked frame chain + // (virtualizable.py:101-113 write_boxes runs under the + // translated write barrier). Arm whichever side the GC + // owns: a GC array re-traces its own items; a stationary + // block is re-walked through the owning frame's custom + // trace. + if majit_gc::gc_owns_object(array_ptr as usize) { + majit_gc::gc_write_barrier(majit_ir::GcRef(array_ptr as usize)); + } else if majit_gc::gc_owns_object(obj_ptr as usize) { + majit_gc::gc_write_barrier(majit_ir::GcRef(obj_ptr as usize)); + } } _ => { let ptr = array_ptr.add(item_offset) as *mut i64; diff --git a/pyre/pyre-interpreter/src/error.rs b/pyre/pyre-interpreter/src/error.rs index 9925ffe3a8d..e5118611d78 100644 --- a/pyre/pyre-interpreter/src/error.rs +++ b/pyre/pyre-interpreter/src/error.rs @@ -411,6 +411,11 @@ impl PyError { forward(&mut self.exc_object); forward(&mut self.w_name_context); forward(&mut self.w_obj_context); + // The exception carrier is non-moving (stable/malloc_typed), so root + // visitors no-op on it during a minor and never reach its fields; + // forward the raw child slots so young tracebacks/args parked across + // a collection stay valid. + unsafe { crate::eval::walk_raw_exception_roots(self.exc_object, visitor) }; } } diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index dce4bc0ccac..c346a3e9e35 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -3859,6 +3859,28 @@ fn live_frame_array_values( .collect() } +/// Re-arm the write barrier on a frame + its `locals_cells_stack_w` array +/// after a JIT-side flush stored refs into them raw. The stored values may +/// be nursery-young while the frame/array are old-gen, and after the flush +/// the frame can run detached from the walked frame chain (virtualizable), +/// so no minor re-traces the items unless an owner sits in the remembered +/// set (virtualizable.py:136 `write_boxes` stores run under the translated +/// write barrier; `remember_frame_locals_array` is the creation-time twin). +/// A GC-owned array re-traces its own items; a stationary `std::alloc` +/// array is only re-walked through its owning frame's custom trace — arm +/// whichever side the GC owns. +pub(crate) fn frame_array_write_barrier( + frame: *mut u8, + arr_ptr: *mut pyre_object::FixedObjectArray, +) { + if pyre_object::gc_hook::try_gc_owns_object(arr_ptr as *mut u8) { + pyre_object::gc_hook::try_gc_write_barrier(arr_ptr as *mut u8); + } + if pyre_object::gc_hook::try_gc_owns_object(frame) { + pyre_object::gc_hook::try_gc_write_barrier(frame); + } +} + /// Write one decoded Ref back into the live frame's locals_cells_stack_w /// (the GC-rooted virtualizable array), re-asserting the resume-decoded /// vable image after the guard-failure vsd correction cleared root slots @@ -3881,6 +3903,7 @@ fn store_live_frame_array_slot(vable_ptr: usize, slot: usize, value: majit_ir::V return; } arr.as_mut_slice()[slot] = r.as_usize() as pyre_object::PyObjectRef; + frame_array_write_barrier(vable_ptr as *mut u8, lp); } /// pyframe.py:107-110: `locals_cells_stack_w` length = @@ -4291,6 +4314,7 @@ fn flush_walk_end_state_to_frame_inner( pf.last_instr = body_pc as isize - 1; } } + frame_array_write_barrier(frame as *mut u8, arr_ptr); true } @@ -4424,6 +4448,7 @@ pub(crate) fn flush_walk_end_state_at_outer_call( pf.valuestackdepth = end_vsd; pf.last_instr = call_py_pc as isize - 1; } + frame_array_write_barrier(frame as *mut u8, arr_ptr); true } @@ -4522,6 +4547,7 @@ pub(crate) fn write_back_outer_locals(ctx: &TraceCtx, frame: usize) -> bool { (*arr_ptr).as_mut_slice()[abs] = boxed; } } + frame_array_write_barrier(frame as *mut u8, arr_ptr); true } @@ -4564,6 +4590,7 @@ pub(crate) fn flush_walk_end_state_after_outer_call( pf.valuestackdepth = nlocals + 1; pf.last_instr = post_call_py_pc as isize - 1; } + frame_array_write_barrier(frame as *mut u8, arr_ptr); true } diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 61866639b93..86e7e3b8428 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -139,16 +139,22 @@ pub fn walk_active_sym_exc_roots(visitor: &mut dyn FnMut(&mut majit_ir::GcRef)) // forwarded pointer back — matching the accepted `jit_driver_pair_from_root_area` // convention in `pyre-jit`. let sym = unsafe { &*sym_ptr }; - let mut mark = |p: pyre_object::PyObjectRef| { - if !p.is_null() { - let mut gcref = majit_ir::GcRef(p as usize); - visitor(&mut gcref); + let carriers = [sym.last_exc_value, sym.current_exc_value]; + for p in carriers + .into_iter() + .chain(sym.trace_built_exc.values().copied()) + { + if p.is_null() { + continue; } - }; - mark(sym.last_exc_value); - mark(sym.current_exc_value); - for exc in sym.trace_built_exc.values() { - mark(*exc); + let mut gcref = majit_ir::GcRef(p as usize); + visitor(&mut gcref); + // The carrier is non-moving, so a minor's root visitor no-ops on it + // and never reaches its fields: young children (tracebacks/args built + // while tracing) would be left dangling across the minor. Forward the + // raw child slots explicitly; the writes land in the exception + // object, not the sym, so the shared-read contract above holds. + unsafe { pyre_interpreter::eval::walk_raw_exception_roots(p, visitor) }; } } @@ -486,7 +492,20 @@ fn try_commit_midbody_abort( }; frame.locals_w_mut().as_mut_slice()[stack_base + rel] = *value; } + // The array is old-gen from birth (`FrameLocalsArrayAllocation::OldGenGc`) + // and `FrameLocalsRoot` only forwards the field slot, not the items: the + // young refs just stored need the remembered set to survive the boxing + // allocations below, and each minor consumes the entry, so re-arm after + // every batch that follows a possible collection. + crate::state::frame_array_write_barrier( + frame.as_mut_ptr() as *mut u8, + frame.locals_w_mut() as *mut _, + ); for (slot, value) in current.live_locals.iter().enumerate() { + crate::state::frame_array_write_barrier( + frame.as_mut_ptr() as *mut u8, + frame.locals_w_mut() as *mut _, + ); frame.locals_w_mut().as_mut_slice()[slot] = match value { None => pyre_object::PY_NULL, Some(crate::state::ConcreteValue::Ref(value)) => *value, @@ -499,6 +518,10 @@ fn try_commit_midbody_abort( } }; } + crate::state::frame_array_write_barrier( + frame.as_mut_ptr() as *mut u8, + frame.locals_w_mut() as *mut _, + ); frame.valuestackdepth = stack_base + current.live_stack.len(); frame.last_instr = words.callee_py_pc as isize - 1; let sys_exc_value_pre = unsafe { (*ec).sys_exc_value }; diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 25d6666dae5..4032a26b1f5 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -3228,11 +3228,41 @@ fn walk_jit_exc_value(visitor: &mut dyn FnMut(&mut majit_ir::GcRef)) { } // A GC-managed exception (post-#18) marked here has its registered child // offsets traced by the collector; the exception is oldgen-stable so a - // bare mark suffices. (The off-GC child-walk fallback is intentionally - // omitted — it only matters with no collector installed, and reaching the - // private `walk_raw_exception_roots` from this crate is unnecessary.) + // bare mark suffices. let mut gcref = majit_ir::GcRef(exc as usize); visitor(&mut gcref); + // The carrier is non-moving (oldgen-stable / malloc_typed), so a minor + // collection's root visitor no-ops on it and never reaches its fields + // (`drag_out_root` returns for non-nursery starts): young children — + // tracebacks appended by the raise in flight, args — would be left + // dangling inside the exception across the minor. Forward the raw child + // slots explicitly, as the EC root walk does for `sys_exc_value`. + unsafe { + pyre_interpreter::eval::walk_raw_exception_roots( + gcref.0 as pyre_object::PyObjectRef, + visitor, + ) + }; +} + +/// Root the blackhole-published exception (`BH_LAST_EXC_VALUE`): a can-raise +/// blackhole helper parks the exception's raw pointer there until the +/// dispatcher drains it (`bhimpl_abort_permanent` / handler dispatch); in that +/// window the exception is reachable only through the raw TLS `i64`. Same +/// carrier/children split as [`walk_jit_exc_value`]. +fn walk_bh_last_exc_value(visitor: &mut dyn FnMut(&mut majit_ir::GcRef)) { + let exc = majit_metainterp::blackhole::BH_LAST_EXC_VALUE.with(|c| c.get()); + if exc == 0 { + return; + } + let mut gcref = majit_ir::GcRef(exc as usize); + visitor(&mut gcref); + unsafe { + pyre_interpreter::eval::walk_raw_exception_roots( + gcref.0 as pyre_object::PyObjectRef, + visitor, + ) + }; } /// Phase B: root walkers that reference interpreter state (immortal dicts, @@ -3241,6 +3271,7 @@ fn walk_jit_exc_value(visitor: &mut dyn FnMut(&mut majit_ir::GcRef)) { fn install_gc_root_walkers() { pyre_interpreter::eval::register_pyframe_root_walker(); majit_gc::shadow_stack::register_extra_root_walker(walk_jit_exc_value); + majit_gc::shadow_stack::register_extra_root_walker(walk_bh_last_exc_value); // Stored `PyError` carriers whose GC refs the precise collector cannot // reach through their raw TLS cells: the call-assembler FFI stash and the // no-handler trace→portal stash. Mirrors `walk_pending_call_error`. @@ -6976,6 +7007,23 @@ fn execute_assembler( ); } + // The interpreter may have left nursery-young refs in the frame's + // locals/stack array; compiled execution runs the virtualizable detached + // from the walked frame chain, so no minor re-traces those items unless + // the frame/array sits in the remembered set. Arm it once at entry + // (upstream every interpreter store runs under the translated write + // barrier, so the array is always covered there). + { + let f = frame_root.frame() as *mut PyFrame as *mut u8; + let arr = unsafe { (*(f as *mut PyFrame)).locals_cells_stack_w }; + if pyre_object::gc_hook::try_gc_owns_object(arr as *mut u8) { + pyre_object::gc_hook::try_gc_write_barrier(arr as *mut u8); + } + if pyre_object::gc_hook::try_gc_owns_object(f) { + pyre_object::gc_hook::try_gc_write_barrier(f); + } + } + // warmstate.py:395 func_execute_token(loop_token, *args) → deadframe let outcome = { let _frame_locals_root = FrameLocalsRoot::new(frame_root.frame()); From 4360175f4785dc36bf71157bf3028a5f8d3fa59c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 24 Jul 2026 00:56:33 +0900 Subject: [PATCH 06/11] jit: enable the exception-edge bridge by default on native backends PYRE_EXC_EDGE_BRIDGE becomes opt-out (=0 disables) on native targets; the wasm guest keeps the opt-in gate (no env plumbing to switch it back off, and its abort-replay exception class is still open). Native jitstats baselines regenerated: every bench's top-level print loop previously hit the pending-exception decline and now compiles (+1 loop, +1 guard failure); fib_recursive additionally converges two GuardNoException bridges (bridges 1->3, guard_failures 1->407, absorbed in warmup). loops_aborted / internal_compile_panics stay 0 everywhere. wasm baselines unchanged. A/B (same binary, env toggle, alternating x3): exc_mixed_classes_ bridge_flavor 0.415s -> 0.166s; handler_reraise_second_exc ~6% faster; no bench regressed. check.py 298/298 on dynasm and cranelift with the default on; adversarial exc battery matches CPython both with the default and with =0. Assisted-by: Claude --- pyre/bench/fannkuch.cranelift.jitstats | 4 ++-- pyre/bench/fannkuch.dynasm.jitstats | 4 ++-- pyre/bench/fib_loop.cranelift.jitstats | 4 ++-- pyre/bench/fib_loop.dynasm.jitstats | 4 ++-- pyre/bench/fib_recursive.cranelift.jitstats | 6 +++--- pyre/bench/fib_recursive.dynasm.jitstats | 6 +++--- pyre/bench/float_loop.cranelift.jitstats | 4 ++-- pyre/bench/float_loop.dynasm.jitstats | 4 ++-- pyre/bench/inline_helper.cranelift.jitstats | 4 ++-- pyre/bench/inline_helper.dynasm.jitstats | 4 ++-- pyre/bench/int_loop.cranelift.jitstats | 4 ++-- pyre/bench/int_loop.dynasm.jitstats | 4 ++-- pyre/bench/nbody.cranelift.jitstats | 4 ++-- pyre/bench/nbody.dynasm.jitstats | 4 ++-- pyre/bench/nested_loop.cranelift.jitstats | 4 ++-- pyre/bench/nested_loop.dynasm.jitstats | 4 ++-- .../bench/raise_catch_loop.cranelift.jitstats | 4 ++-- pyre/bench/raise_catch_loop.dynasm.jitstats | 4 ++-- pyre/bench/spectral_norm.cranelift.jitstats | 4 ++-- pyre/bench/spectral_norm.dynasm.jitstats | 4 ++-- .../src/jitcode_dispatch/mod.rs | 21 +++++++++++++------ 21 files changed, 57 insertions(+), 48 deletions(-) diff --git a/pyre/bench/fannkuch.cranelift.jitstats b/pyre/bench/fannkuch.cranelift.jitstats index 32e10c2c07c..c295bcd9dac 100644 --- a/pyre/bench/fannkuch.cranelift.jitstats +++ b/pyre/bench/fannkuch.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=40 -guard_failures=8202 +guard_failures=8203 internal_compile_panics=0 loops_aborted=0 -loops_compiled=5 +loops_compiled=6 diff --git a/pyre/bench/fannkuch.dynasm.jitstats b/pyre/bench/fannkuch.dynasm.jitstats index 32e10c2c07c..c295bcd9dac 100644 --- a/pyre/bench/fannkuch.dynasm.jitstats +++ b/pyre/bench/fannkuch.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=40 -guard_failures=8202 +guard_failures=8203 internal_compile_panics=0 loops_aborted=0 -loops_compiled=5 +loops_compiled=6 diff --git a/pyre/bench/fib_loop.cranelift.jitstats b/pyre/bench/fib_loop.cranelift.jitstats index c868dc401e3..ec832e2115d 100644 --- a/pyre/bench/fib_loop.cranelift.jitstats +++ b/pyre/bench/fib_loop.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=189 +guard_failures=190 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/fib_loop.dynasm.jitstats b/pyre/bench/fib_loop.dynasm.jitstats index c868dc401e3..ec832e2115d 100644 --- a/pyre/bench/fib_loop.dynasm.jitstats +++ b/pyre/bench/fib_loop.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=189 +guard_failures=190 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/fib_recursive.cranelift.jitstats b/pyre/bench/fib_recursive.cranelift.jitstats index 87381d9b65c..a5decaa2b0d 100644 --- a/pyre/bench/fib_recursive.cranelift.jitstats +++ b/pyre/bench/fib_recursive.cranelift.jitstats @@ -1,5 +1,5 @@ -bridges_compiled=1 -guard_failures=1 +bridges_compiled=3 +guard_failures=407 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/fib_recursive.dynasm.jitstats b/pyre/bench/fib_recursive.dynasm.jitstats index 87381d9b65c..a5decaa2b0d 100644 --- a/pyre/bench/fib_recursive.dynasm.jitstats +++ b/pyre/bench/fib_recursive.dynasm.jitstats @@ -1,5 +1,5 @@ -bridges_compiled=1 -guard_failures=1 +bridges_compiled=3 +guard_failures=407 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/float_loop.cranelift.jitstats b/pyre/bench/float_loop.cranelift.jitstats index 21782197f44..6142c4e8679 100644 --- a/pyre/bench/float_loop.cranelift.jitstats +++ b/pyre/bench/float_loop.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=1 +guard_failures=2 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/float_loop.dynasm.jitstats b/pyre/bench/float_loop.dynasm.jitstats index 21782197f44..6142c4e8679 100644 --- a/pyre/bench/float_loop.dynasm.jitstats +++ b/pyre/bench/float_loop.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=1 +guard_failures=2 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/inline_helper.cranelift.jitstats b/pyre/bench/inline_helper.cranelift.jitstats index 21782197f44..6142c4e8679 100644 --- a/pyre/bench/inline_helper.cranelift.jitstats +++ b/pyre/bench/inline_helper.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=1 +guard_failures=2 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/inline_helper.dynasm.jitstats b/pyre/bench/inline_helper.dynasm.jitstats index 21782197f44..6142c4e8679 100644 --- a/pyre/bench/inline_helper.dynasm.jitstats +++ b/pyre/bench/inline_helper.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=1 +guard_failures=2 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/int_loop.cranelift.jitstats b/pyre/bench/int_loop.cranelift.jitstats index 21782197f44..6142c4e8679 100644 --- a/pyre/bench/int_loop.cranelift.jitstats +++ b/pyre/bench/int_loop.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=1 +guard_failures=2 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/int_loop.dynasm.jitstats b/pyre/bench/int_loop.dynasm.jitstats index 21782197f44..6142c4e8679 100644 --- a/pyre/bench/int_loop.dynasm.jitstats +++ b/pyre/bench/int_loop.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=0 -guard_failures=1 +guard_failures=2 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/nbody.cranelift.jitstats b/pyre/bench/nbody.cranelift.jitstats index 721d05d05d2..6816842e5e9 100644 --- a/pyre/bench/nbody.cranelift.jitstats +++ b/pyre/bench/nbody.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=5 -guard_failures=1286 +guard_failures=1287 internal_compile_panics=0 loops_aborted=0 -loops_compiled=5 +loops_compiled=6 diff --git a/pyre/bench/nbody.dynasm.jitstats b/pyre/bench/nbody.dynasm.jitstats index 721d05d05d2..6816842e5e9 100644 --- a/pyre/bench/nbody.dynasm.jitstats +++ b/pyre/bench/nbody.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=5 -guard_failures=1286 +guard_failures=1287 internal_compile_panics=0 loops_aborted=0 -loops_compiled=5 +loops_compiled=6 diff --git a/pyre/bench/nested_loop.cranelift.jitstats b/pyre/bench/nested_loop.cranelift.jitstats index aaea7bf611d..5a4453f26d2 100644 --- a/pyre/bench/nested_loop.cranelift.jitstats +++ b/pyre/bench/nested_loop.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=1 -guard_failures=201 +guard_failures=202 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/nested_loop.dynasm.jitstats b/pyre/bench/nested_loop.dynasm.jitstats index aaea7bf611d..5a4453f26d2 100644 --- a/pyre/bench/nested_loop.dynasm.jitstats +++ b/pyre/bench/nested_loop.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=1 -guard_failures=201 +guard_failures=202 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/raise_catch_loop.cranelift.jitstats b/pyre/bench/raise_catch_loop.cranelift.jitstats index aaea7bf611d..5a4453f26d2 100644 --- a/pyre/bench/raise_catch_loop.cranelift.jitstats +++ b/pyre/bench/raise_catch_loop.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=1 -guard_failures=201 +guard_failures=202 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/raise_catch_loop.dynasm.jitstats b/pyre/bench/raise_catch_loop.dynasm.jitstats index aaea7bf611d..5a4453f26d2 100644 --- a/pyre/bench/raise_catch_loop.dynasm.jitstats +++ b/pyre/bench/raise_catch_loop.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=1 -guard_failures=201 +guard_failures=202 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 diff --git a/pyre/bench/spectral_norm.cranelift.jitstats b/pyre/bench/spectral_norm.cranelift.jitstats index 8ae1dd2f977..4f8074a1464 100644 --- a/pyre/bench/spectral_norm.cranelift.jitstats +++ b/pyre/bench/spectral_norm.cranelift.jitstats @@ -1,5 +1,5 @@ bridges_compiled=2 -guard_failures=441 +guard_failures=442 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=4 diff --git a/pyre/bench/spectral_norm.dynasm.jitstats b/pyre/bench/spectral_norm.dynasm.jitstats index 8ae1dd2f977..4f8074a1464 100644 --- a/pyre/bench/spectral_norm.dynasm.jitstats +++ b/pyre/bench/spectral_norm.dynasm.jitstats @@ -1,5 +1,5 @@ bridges_compiled=2 -guard_failures=441 +guard_failures=442 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=4 diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index fe905c58e98..f92abc642f1 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -2359,14 +2359,23 @@ pub(crate) fn try_catch_exception_at(code: &[u8], position: usize) -> Option bool { static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| std::env::var_os("PYRE_EXC_EDGE_BRIDGE").is_some()) + *ENABLED.get_or_init(|| { + if cfg!(target_arch = "wasm32") { + return std::env::var_os("PYRE_EXC_EDGE_BRIDGE").is_some(); + } + match std::env::var_os("PYRE_EXC_EDGE_BRIDGE") { + Some(v) => v != "0", + None => true, + } + }) } /// `PYRE_CARRIER_EXC_RESUME=1` enables the multi-frame (carrier) exception From 44e17edc49ad9f93cc20cb482264dc56a824fbb2 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 24 Jul 2026 03:14:52 +0900 Subject: [PATCH 07/11] jit: remove the served-their-purpose default-on FBW rollout flags Every flag below was default-on with an unused `=0` opt-out. Delete the gate machinery and make the enabled behavior unconditional; the disabled arms and their helper code are removed as dead. Removed env flags (collapsed to always-on): PYRE_FBW_INLINE, _INLINE_MULTIFRAME, _NSVABLE_MULTIFRAME, _REC_MULTIFRAME, _BRIDGE_REC_INLINE, _REC_MUTUAL_CUTOVER, _REC_CA, _FORITER_INLINE, _LOOP_CALLEE_CA, _RAISE, _BUILTIN_FOLD, _LOADATTR_FOLD, _STOREATTR_FOLD, _LOADMETHOD_FOLD, _LOADGLOBAL_FOLD, _LOADNAME_FOLD, _STORENAME_FOLD, _DELETE_FAST, _INLINE_NSFOLD, _STACK_LIVEREG, _CALL_ASSEMBLER, _NO_REPLAY_EXIT, _NESTED_RESID_ABORT, _ABORT_FLUSH, _BRANCH_FLUSH, _END_FLUSH, _BRIDGE_STAMP, _BRIDGE_LOCAL_SEED. exc_edge_bridge_enabled() becomes `cfg!(not(target_arch = "wasm32"))`: native backends run the exception-edge bridge unconditionally; the wasm guest's abort-replay exception class (#727) is still open, so it stays off there. check.py: drop the --no-fbw-inline-multiframe option and its PYRE_FBW_INLINE_MULTIFRAME=0 export. Assisted-by: Claude --- majit/majit-ir/src/effectinfo.rs | 6 +- ...me_escape_flush_writethrough_regression.py | 2 +- pyre/bench/synth/bridge_recursion_overflow.py | 2 +- .../synth/foriter_exempt_nested_foriter.py | 2 +- .../synth/foriter_exempt_shared_generator.py | 2 +- pyre/check.py | 18 - pyre/gate-triage.md | 5 - .../src/jitcode_dispatch/bridge_subwalk.rs | 2 +- .../src/jitcode_dispatch/fbw_state.rs | 326 +-------- .../src/jitcode_dispatch/inline_call.rs | 101 +-- .../src/jitcode_dispatch/mod.rs | 391 ++++------ .../src/jitcode_dispatch/residual_call.rs | 69 +- .../src/jitcode_dispatch/resume_snapshot.rs | 54 +- .../src/jitcode_dispatch/specialize.rs | 24 +- .../src/jitcode_dispatch/tests.rs | 6 +- .../src/jitcode_dispatch/vable_ops.rs | 6 +- pyre/pyre-jit-trace/src/state.rs | 68 +- pyre/pyre-jit-trace/src/trace.rs | 690 ++++++++---------- pyre/pyre-jit/src/call_jit.rs | 4 +- pyre/pyre-jit/src/jit/codewriter.rs | 345 ++++----- 20 files changed, 791 insertions(+), 1332 deletions(-) diff --git a/majit/majit-ir/src/effectinfo.rs b/majit/majit-ir/src/effectinfo.rs index 80c9d97e525..ac5d4649619 100644 --- a/majit/majit-ir/src/effectinfo.rs +++ b/majit/majit-ir/src/effectinfo.rs @@ -603,13 +603,13 @@ pub enum PyreHelperKind { /// without `from` (checked, never dereferenced when null), so a concrete /// NULL there is the normal shape — not the broken baked-NULL shape the /// walker's may-force NULL-ref gate rejects. The full-body walker - /// recognises this tag (gated `PYRE_FBW_RAISE`) to exempt the trailing + /// recognises this tag to exempt the trailing /// NULL `cause` in both twin NULL-ref guards so the FBW path can own the /// raise instead of declining to the trait. RaiseVarargs, /// `get_current_exception()` — the PUSH_EXC_INFO `prev = ec.sys_exc_value` /// save residual (`() → Ref`, TLS read via `cpu.get_current_exception_fn`). - /// The full-body walker recognises this tag (gated `PYRE_FBW_RAISE`) to + /// The full-body walker recognises this tag to /// lower it to `GETFIELD_GC_R(ec, sys_exc_value)` so the exc-info save /// participates in the balanced save/restore the heap optimizer /// dead-store-eliminates, letting a non-escaping exception stay virtual. @@ -617,7 +617,7 @@ pub enum PyreHelperKind { /// `set_current_exception(exc)` — the PUSH_EXC_INFO store and the /// POP_EXCEPT restore residual (`(exc: Ref) → Void`, TLS write via /// `cpu.set_current_exception_fn`). The full-body walker recognises this - /// tag (gated `PYRE_FBW_RAISE`) to lower it to `SETFIELD_GC(ec, exc, + /// tag to lower it to `SETFIELD_GC(ec, exc, /// sys_exc_value)`; paired with the [`GetCurrentException`] save on the /// same descr-identity field, a balanced never-read save/restore is /// dead-store-eliminated so the virtual exception de-escapes and DCEs. diff --git a/pyre/bench/getframe_escape_flush_writethrough_regression.py b/pyre/bench/getframe_escape_flush_writethrough_regression.py index cda6e37a638..b70f29826ad 100644 --- a/pyre/bench/getframe_escape_flush_writethrough_regression.py +++ b/pyre/bench/getframe_escape_flush_writethrough_regression.py @@ -5,7 +5,7 @@ # write-through. Reading the redirected caller frame forces it mid-expression: # the escape flush must commit with the operand-stack mirror (the vable shadow's # stack region is NULL there) and resume forward AT the escaping opcode. The -# legacy replay-from-loop-entry fallback (PYRE_FBW_ABORT_FLUSH=0) drops the +# legacy replay-from-loop-entry fallback drops the # in-flight FOR_ITER iteration instead, so `total` comes up short -- the JIT-only # regression this guards. # diff --git a/pyre/bench/synth/bridge_recursion_overflow.py b/pyre/bench/synth/bridge_recursion_overflow.py index f1dd3598735..fd909627823 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.py +++ b/pyre/bench/synth/bridge_recursion_overflow.py @@ -1,5 +1,5 @@ # Parity fixture for the depth-1 bridge self-recursive inline lift -# (PYRE_FBW_BRIDGE_REC_INLINE, default on, #704). `f` is a tail-recursive +# (#704). `f` is a tail-recursive # exact-integer callee whose `acc * 2 + 1` crosses the machine-int boundary # partway down the recursion, so an overflow guard fires inside the frame the # lift inlines on a guard-failure bridge. #704's A/B ran on `fib`, which never diff --git a/pyre/bench/synth/foriter_exempt_nested_foriter.py b/pyre/bench/synth/foriter_exempt_nested_foriter.py index 89093efa0f9..b7cf076c15d 100644 --- a/pyre/bench/synth/foriter_exempt_nested_foriter.py +++ b/pyre/bench/synth/foriter_exempt_nested_foriter.py @@ -1,4 +1,4 @@ -# gh#495 guard: ForIterNext exemption double-advance is masked by fbw_abort_nested_unjournaled_residual; +1 reproduces under PYRE_FBW_NESTED_RESID_ABORT=0 +# gh#495 guard: fbw_abort_nested_unjournaled_residual prevents the ForIterNext exemption double-advance. # branch-bearing callee with a SECOND FOR_ITER (nested), not the loop header. # Two shared generators; inner FOR_ITER advance is a non-header foriter (Finding #2). # Post-inner declining residual forces abort while inner item in-flight. diff --git a/pyre/bench/synth/foriter_exempt_shared_generator.py b/pyre/bench/synth/foriter_exempt_shared_generator.py index 38eb97ddb21..205fff6bcff 100644 --- a/pyre/bench/synth/foriter_exempt_shared_generator.py +++ b/pyre/bench/synth/foriter_exempt_shared_generator.py @@ -1,4 +1,4 @@ -# gh#495 guard: ForIterNext exemption double-advance is masked by fbw_abort_nested_unjournaled_residual; +1 reproduces under PYRE_FBW_NESTED_RESID_ABORT=0 +# gh#495 guard: fbw_abort_nested_unjournaled_residual prevents the ForIterNext exemption double-advance. # SHARED long generator consumed incrementally. step consumes ONE item (for..break), # FOR_ITER advance mutates shared counter (exempt). Then a declining nested-residual CALL. # If the inline sub-walk aborts AFTER the exempt advance and the trait leg re-runs step, diff --git a/pyre/check.py b/pyre/check.py index dc6480a8306..7d3c52853de 100644 --- a/pyre/check.py +++ b/pyre/check.py @@ -67,12 +67,6 @@ def _detect_pyre_stdlib(): PYRE_STDLIB = _detect_pyre_stdlib() -# Opt-out (`--no-fbw-inline-multiframe`): export PYRE_FBW_INLINE_MULTIFRAME=0 into -# pyre child runs to exercise the #68 multi-frame inline rollback escape hatch. -# The path is on by default, so the default run already parity-checks it; this -# opt-out validates the flag-off fallback. -FBW_INLINE_MULTIFRAME_OFF = False - # Which wasm runtime the `pyre-wasm-runner` uses (`--wasm-engine`). wasmtime # (cranelift) is fast in steady state but recompiles the ~14MB module on every # process start; wasmi is a pure-Rust interpreter with near-zero startup cost @@ -304,8 +298,6 @@ def pyre_env(): # in the environment wins. if PYRE_STDLIB and "PYRE_STDLIB" not in env: env["PYRE_STDLIB"] = PYRE_STDLIB - if FBW_INLINE_MULTIFRAME_OFF: - env["PYRE_FBW_INLINE_MULTIFRAME"] = "0" # Point the wasm runner at the built module by absolute path so it resolves # regardless of the child's working directory (ignored by other backends). if "PYRE_WASM_MODULE" not in env and Path(WASM_MODULE_PATH).exists(): @@ -1663,13 +1655,6 @@ def parse_backend_specs(specs): default=20.0, help="per-script timeout in seconds for synthetic benchmarks", ) - parser.add_argument( - "--no-fbw-inline-multiframe", - action="store_true", - help="run pyre with PYRE_FBW_INLINE_MULTIFRAME=0 (#68 forward-branch " - "multi-frame inline is on by default; this exercises the rollback " - "escape hatch)", - ) parser.add_argument("pyre_path", nargs="?", default="") args = parser.parse_args() try: @@ -1697,9 +1682,6 @@ def parse_backend_specs(specs): def main(): args = parse_args() - if args.no_fbw_inline_multiframe: - global FBW_INLINE_MULTIFRAME_OFF - FBW_INLINE_MULTIFRAME_OFF = True global WASM_ENGINE WASM_ENGINE = args.wasm_engine chk = Check(args) diff --git a/pyre/gate-triage.md b/pyre/gate-triage.md index 48164e130d1..c87a6f37127 100644 --- a/pyre/gate-triage.md +++ b/pyre/gate-triage.md @@ -143,11 +143,6 @@ OFF path is a needed safety net. Retire at the listed trigger (A7). | var | subsystem | retire when | |---|---|---| -| PYRE_FBW_INLINE, _INLINE_MULTIFRAME, _INLINE_NSFOLD, _LOOP_CALLEE_CA | walker inlining (#62/#68/gap-10) | same epic cluster | -| PYRE_FBW_CALL_ASSEMBLER, _NO_REPLAY_EXIT, _RAISE, _REC_CA | walker return/raise/recursion | same | -| PYRE_FBW_ABORT_FLUSH, _BRANCH_FLUSH, _END_FLUSH, _BRIDGE_LOCAL_SEED | shadow-stack flush/seed on resume | same (couples to F1 resume convergence) | -| PYRE_FBW_BUILTIN_FOLD, _LOADGLOBAL_FOLD, _LOADNAME_FOLD, _STORENAME_FOLD | const-folds in walker bodies | same (fold correctness interlocks with the walker) | -| PYRE_FBW_NESTED_RESID_ABORT | nested-residual abort vs replay | same | | PYRE_TWO_PHASE_RTYPE, PYRE_TUPLE_PER_SHAPE_CLASSDEF | rtyper prepass / per-shape tuple classdef | WS2 / #346 rtyper epic | | PYRE_ORIGINAL_BOXES | greens++reds original_boxes index shape | box-identity #202 / resume F1 | | PYRE_MIR_FRAMESTATE | framestate-threaded MIR lowering | MIR front-end #176/#181/#346 | diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index d5e4ae7b23e..3f970321902 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -195,7 +195,7 @@ pub fn dispatch_via_miframe( ConcreteValue::Ref(sym.last_exc_value()) }; - // Exception-edge bridge routing (`PYRE_EXC_EDGE_BRIDGE`): an exception-guard + // Exception-edge bridge routing: an exception-guard // bridge with a standing exception resumes at the no-exception fallthrough // `-live-`, NOT the `except` handler. Mirror the blackhole // `handle_exception_in_frame` backward case: route the walk entry to the diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index 53d67c4ebf6..53c57502220 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -95,148 +95,7 @@ pub(crate) fn fbw_strict_fold_frame_reg(ctx: &WalkContext<'_, '_, .map_or(u16::MAX, |shadow| shadow.fold_frame_reg) } -/// `PYRE_FBW_INLINE_MULTIFRAME` (#68): inline branch-bearing callees with a -/// multi-frame guard snapshot instead of declining them to interpretation -/// (`LoopBearingCalleeInlineUnsupported`). Default-on; `PYRE_FBW_INLINE_MULTIFRAME=0` -/// (or `false`) is the rollback escape hatch. The multi-frame snapshot -/// encode↔decode contract for walker-emitted callee-frame guards is validated -/// byte-exact (function_calls + corpus) on both backends. -pub(crate) fn fbw_inline_multiframe_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_INLINE_MULTIFRAME") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_NSVABLE_MULTIFRAME` (#73): publish the `_nonstandard_virtualizable` -/// promote guard through the full multi-frame resume chain (each paused caller -/// plus the callee's own coordinate) instead of the single-frame sentinel -/// collapse in [`walker_capture_inline_nonstandard_vable_guard`], which cannot -/// resolve a JitCode resume word and unconditionally aborts every inline -/// sub-walk emit of this guard. Default-on; `PYRE_FBW_NSVABLE_MULTIFRAME=0` -/// (or `false`) restores the sentinel decline as the rollback escape hatch. -pub(crate) fn fbw_nsvable_multiframe_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_NSVABLE_MULTIFRAME") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_REC_MULTIFRAME` (default ON; `=0`/`false` opts out): route -/// primary-trace self-recursive Python calls through the multiframe inline path -/// while below `PYRE_FBW_MULTIFRAME_DEPTH`, instead of folding immediately to -/// the recursive portal `CALL_ASSEMBLER`. -/// -/// RPython parity: `opimpl_recursive_call` / `do_recursive_call` -/// (`pyjitpl.py`) inline within `max_unroll_recursion`; only once the -/// cap is reached does recursion fall back to the assembler-call path. The -/// prior fold-only default (every self-recursive call cut straight to -/// `CALL_ASSEMBLER`) was the pyre deviation; inlining below the depth bound is -/// the parity behavior, so this is default-on. The depth bound -/// (`fbw_max_multiframe_depth`, default 1) still caps how deep the inline -/// unrolls before falling back to `CALL_ASSEMBLER`. -pub(crate) fn fbw_rec_multiframe_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_REC_MULTIFRAME") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_BRIDGE_REC_INLINE` (default ON) — on a plain root bridge walk, -/// lift the bridge-trace decline that keeps an exact-integer arithmetic callee a -/// single residual call, letting the bridge inline one self-recursive level -/// exactly as a primary trace does: the call falls through to the self-recursive -/// unroll gate and the multiframe seed instead of returning a residual. The -/// miscompile hazard it admits — a bridge-inlined int-binop callee's second -/// virtual frame operand stack has no red bridge input, so an overflow/exception -/// resume path can leave a NULL vable stack slot — is contained by the -/// seed-success precondition plus the `n_parents == n_callees` snapshot valve -/// fallbacks. A/B across the bench corpus is byte-parity clean and adds no -/// `loops_aborted` or `internal_compile_panics`; fib_recursive gains one inlined -/// bridge (guard_failures 407 -> 406, bridges_compiled 2 -> 3) for a ~10% win. -/// `=0`/`false` opts back out. -pub(crate) fn fbw_bridge_rec_inline_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_BRIDGE_REC_INLINE") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// Full-portal recursive-call cutover (`PYRE_FBW_REC_MUTUAL_CUTOVER`): at the -/// inline-unroll cap, route a recursive callee (self OR mutual) through -/// `get_assembler_token` → `compile_tmp_callback` (warmstate.py, -/// compile.py) so a not-yet-compiled callee still enters via a real -/// CALL_ASSEMBLER tmp-callback token instead of poisoning the trace with -/// `LoopBearingCalleeInlineUnsupported`. Mirrors the `build_jit_driver_pair` -/// gate of the same name (eval.rs); default ON, `=0` opts out. -pub(crate) fn fbw_rec_mutual_cutover_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_REC_MUTUAL_CUTOVER") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_LOOP_CALLEE_CA` (general loop-bearing-callee → -/// CALL_ASSEMBLER): when a multi-frame inlined callee sub-walk reaches the -/// callee's own `jit_merge_point` and a compiled loop token already exists -/// for that green key, emit a `CALL_ASSEMBLER` into it (mirror of -/// `opimpl_recursive_call_assembler`) instead of declining the enclosing -/// trace (`JitMergePointGreenKeyUnresolved`). Default-ON; -/// `=0`/`false` opts out. -/// -/// The default-ON flip rides the same CALL_ASSEMBLER / residual-executor / -/// virtualizable machinery already shipping default-ON through the -/// self-recursive arm ([`try_walker_call_assembler_self_recursive`], -/// `PYRE_FBW_REC_CA` default-ON). The only extension here is the callee -/// frame shape: a multi-frame inline frame built by -/// `emit_new_pyframe_inline_with_params` that can hold Ref locals, vs the -/// self-recursive arm's int-only `emit_new_pyframe_inline_self_recursive` -/// frame. A four-lens GC-rooting audit established the two frame builders are -/// content-agnostically rooted identically — same `pyframe_size_descr()`, -/// same `pyobject_gcarray_descr()` locals array, same malloc-then-store -/// ordering, the materialized virtualizable frame is JUMP-loop-carried so its -/// slot is in every inner residual-call gcmap (`get_gcmap`), and the runtime -/// `PyFrame`/array GC type registration traces frame->array->elements with no -/// int-vs-ref branch anywhere. A historical GC-stress SEGV (a freed, -/// not-forwarded receiver under nursery pressure) reproduced only on -/// layout-shifting diagnostic-probe builds; on clean binaries it does not -/// reproduce across the GC-stress matrix (r1/r5/r6/r2/r4 × nursery -/// {default,1M,256K,64K,16K,4K} × dynasm+x86, all clean) — consistent with a -/// diagnostic-build layout artifact, and content-agnostic rooting rules out a -/// ref-specific defect in this chain. -pub(crate) fn fbw_loop_callee_ca_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_LOOP_CALLEE_CA") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_VABLE_SCALAR_CA` (default OFF) — sub-mode of -/// [`fbw_loop_callee_ca_enabled`]. When on, the loop-callee +/// `PYRE_FBW_VABLE_SCALAR_CA` (default OFF) — sub-mode of the loop-callee /// CALL_ASSEMBLER passes the callee's loop-carried locals as scalar /// CALL_ASSEMBLER args plus a `VableExpansion` (`arg_overrides` mapping each /// scalar to a callee jitframe slot), so the optimizer can elide the per-call @@ -257,131 +116,6 @@ pub(crate) fn fbw_vable_scalar_ca_enabled() -> bool { }) } -/// `PYRE_FBW_RAISE` (default ON) — the FBW walker owns the Python raise/except -/// loop. The twin NULL-ref guards exempt the trailing `cause` sentinel of a -/// [`PyreHelperKind::RaiseVarargs`] residual so the walker records the raise. -/// Now that the trait tracer is retired, declining instead -/// re-interprets without JIT (a hot raise/except loop would time out), so the -/// walker must own the raise path; `PYRE_FBW_RAISE=0` opts back to declining. -pub(crate) fn fbw_raise_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_RAISE") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_BUILTIN_FOLD` (default ON) — gates the LOAD_GLOBAL cell fold's -/// reachability inside handler-bearing bodies and its builtins-fallback arm. -/// When ON, `dispatch_residual_call_iRd_kind` attempts -/// [`try_walker_load_global_cell_fold`] even when the body contains a -/// `catch_exception` (the fold emits an `ElidableCannotRaise` lookup so the -/// dropped `GUARD_NO_EXCEPTION` is moot for a SUCCESSFUL fold — a declined -/// fold keeps the residual+guard), and the fold resolves names absent from -/// the module dict through `frame.get_builtin()` (e.g. `raise ValueError` / -/// `except ValueError`). `PYRE_FBW_BUILTIN_FOLD=0` restores the legacy -/// handler-free-only behavior. -pub(crate) fn fbw_builtin_fold_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_BUILTIN_FOLD") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_LOADATTR_FOLD` — gate the full-body-walker LOAD_ATTR fast path -/// ([`try_walker_specialize_load_attr`]). When on, a monomorphic plain -/// instance-attribute read folds to guards + inline storage read instead of the -/// opaque `getattr_fn` residual. Default ON (`0`/`false` opts out as the kill -/// switch); verified byte-exact on synth dynasm + cranelift and GC-soak clean -/// under `PYPY_GC_NURSERY=131072` on instance-heavy benches. -pub(crate) fn fbw_loadattr_fold_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_LOADATTR_FOLD") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_STOREATTR_FOLD` — gate the full-body-walker STORE_ATTR fast path -/// ([`try_walker_specialize_store_attr`]). When on, a plain same-type unboxed -/// integer store folds to guards + a non-forcing raw longlong-list write -/// instead of the forcing `setattr_fn` residual (dropping the force token, -/// vable spill, and the value re-box). Default ON (`0`/`false` opts out as the -/// kill switch, independent of the read fold since the write executes a -/// concrete heap mutation). -pub(crate) fn fbw_storeattr_fold_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_STOREATTR_FOLD") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_LOADMETHOD_FOLD` — gate the full-body-walker method-cache fold. -/// When on, a monomorphic `obj.method(...)` dispatch folds the LOAD_ATTR -/// method lookup to a constant descriptor plus guards, and folds the paired -/// `load_method_self` residual to its constant binding decision. Default ON; -/// `0`/`false` opts out as the kill switch. -pub(crate) fn fbw_loadmethod_fold_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_LOADMETHOD_FOLD") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_DELETE_FAST` — gate the full-body-walker DELETE_FAST lowering. -/// Default ON; `0`/`false` opts back into the existing `abort_permanent` -/// marker fallback for unsupported shapes. -pub(crate) fn fbw_delete_fast_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_DELETE_FAST") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - -/// `PYRE_FBW_INLINE_NSFOLD` (default ON) — gates resolving an inlined callee's -/// `getfield_vable_r` namespace(idx5)/pycode(idx1) read from the callee's -/// compile-time [`InlineCalleeConsts`] on the MULTIFRAME path (seeded virtual -/// frame), not just the strict path (unseeded frame). Without it, the seeded -/// virtual frame's vable read misses the heapcache forward — the codewriter's -/// per-fn vable descr identity differs from the seeding descr -/// (`pyframe_w_globals_obj_descr`) — and records a non-const `GetfieldGcR`, -/// leaving the LOAD_GLOBAL fold's namespace operand non-concrete so a -/// loop-bearing inlined callee's `load_global` (e.g. nbody `advance()`'s -/// `len(bodies)`) stays a residual that the nested-unjournaled-residual abort -/// declines. `=0` restores the strict-path-only behavior. -pub(crate) fn fbw_inline_nsfold_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_INLINE_NSFOLD") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - /// `PYRE_FBW_CALLEE_VSTACK` (default OFF) — maintain a callee-local /// operand-stack mirror while walking an inline sub-call. The callee enters /// with an empty operand stack; subsequent boundaries must use the active @@ -397,31 +131,13 @@ pub(crate) fn fbw_callee_vstack_enabled() -> bool { }) } -/// `PYRE_FBW_STACK_LIVEREG` (default ON) — for branch-guard operand-stack -/// snapshot slots, prefer the live Ref register (`pyjitpl.py` -/// `get_list_of_active_boxes` reads `self.registers_r[index]`) when the -/// guard PC's per-PC color map proves that color owns the same stack slot. -/// `=0` restores the shadow-first pyre-local order everywhere. -pub(crate) fn fbw_stack_livereg_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| match std::env::var_os("PYRE_FBW_STACK_LIVEREG") { - Some(v) => { - let v = v.to_string_lossy(); - v != "0" && !v.eq_ignore_ascii_case("false") - } - None => true, - }) -} - thread_local! { - /// Finish payload stashed by a top-level `*_return` arm under the - /// `PYRE_FBW_CALL_ASSEMBLER` gate, read back by + /// Finish payload stashed by a top-level `*_return` arm, read back by /// [`crate::trace::full_body_walk_trace`] to build a /// `TraceAction::Finish` for a loop-free (Finish-terminated) portal. /// /// `(finish_value, finish_arg_type)` — the re-boxed return value and - /// its `Type::Ref` portal-exit type. `None` outside the gated path, - /// so the default-off walk maps `Terminate -> Abort` exactly as before. + /// its `Type::Ref` portal-exit type. /// Reset at the start of every walk (`fbw_finish_payload_reset`) so a /// stale payload from a prior aborted walk cannot leak into this one. static FBW_FINISH_PAYLOAD: std::cell::Cell> = @@ -492,26 +208,6 @@ thread_local! { const { std::cell::Cell::new(false) }; } -/// Whether the Finish-portal compile route is enabled. Cached so -/// the per-`*_return` read and the `full_body_walk_trace` read see a -/// single consistent value. Default ON; `PYRE_FBW_CALL_ASSEMBLER=0` opts -/// back into the pre-Finish-portal path (bare -/// `Terminate` -> `Abort`) as a transition escape hatch. -pub(crate) fn fbw_call_assembler_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| std::env::var("PYRE_FBW_CALL_ASSEMBLER").as_deref() != Ok("0")) -} - -/// Whether the no-replay portal exit is enabled (a loop-free function -/// trace that reached `done_with_this_frame` returns its captured concrete -/// result directly instead of re-running the freshly compiled trace for -/// the SAME invocation). Default ON; `PYRE_FBW_NO_REPLAY_EXIT=0` opts -/// back into the legacy `ContinueRunningNormally` replay for bisection. -pub(crate) fn fbw_no_replay_exit_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| std::env::var("PYRE_FBW_NO_REPLAY_EXIT").as_deref() != Ok("0")) -} - /// Arm/disarm the bridge `Terminate` no-replay shortcut for the next walk /// (see [`FBW_BRIDGE_NOREPLAY_ARMED`]). The bridge tracer sets it before /// the walk and clears it after. @@ -674,11 +370,11 @@ pub(crate) fn fbw_store_journal_reset() { // clears the per-entry body-effect signal so a prior walk's committed // mutation cannot block this walk's delivery. FBW_FORITER_INFLIGHT.with(|c| c.borrow_mut().clear()); - // B3 (`PYRE_FBW_RAISE`): drop any inline-built-exception OpRef keys a + // B3: drop any inline-built-exception OpRef keys a // prior aborted walk recorded, so they cannot match a same-numbered // OpRef minted by this walk's recorder. FBW_BUILT_EXC.with(|s| s.borrow_mut().clear()); - // B3 (`PYRE_FBW_RAISE`): drop any unbalanced PUSH_EXC_INFO prev saves a + // B3: drop any unbalanced PUSH_EXC_INFO prev saves a // prior aborted walk left (an exception that propagated out without its // POP_EXCEPT restore), so a stale saved-prev cannot be popped by an // unrelated POP_EXCEPT in this walk. @@ -1330,13 +1026,6 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( ctx: &WalkContext<'_, '_, Sym>, pc: usize, ) -> Result<(), DispatchError> { - // `PYRE_FBW_NESTED_RESID_ABORT=0` opts back into the prior - // (miscompiling) mark-and-replay behavior for A/B. - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - let enabled = *ENABLED.get_or_init(|| { - std::env::var_os("PYRE_FBW_NESTED_RESID_ABORT").as_deref() - != Some(std::ffi::OsStr::new("0")) - }); // RPython `do_residual_call` runs the residual executor at any framestack // depth (`pyjitpl.py`). Exempt only the self-recursive // `CALL_ASSEMBLER` fold's concrete-stamp executor from this pyre-local @@ -1354,10 +1043,9 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( // `wasm_ca_trampoline_decline` witness). Both are properties of the // framestack knowable at the residual decline point, so the whole trace // aborts before the hazardous body is committed. Every other nested - // residual inlines. The hazard scan is last so the cheap flag checks + // residual inlines. The hazard scan is last so the cheap checks // short-circuit it. - if enabled - && !in_selfrec_fold + if !in_selfrec_fold && !in_exception_string_inline && !ctx.session.borrow().framestack.is_empty() && fbw_inline_callee_hazardous(ctx) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 2c45b7c010e..54f496d295f 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -238,7 +238,7 @@ pub(crate) fn inline_resolvable_static_vable_read( } /// Relaxed variant of [`callee_fast_path_inlinable`] for the multi-frame -/// inline path (#68, `PYRE_FBW_INLINE_MULTIFRAME`): a FORWARD `goto_if_not` +/// inline path (#68): a FORWARD `goto_if_not` /// (branch target ahead of the branch op) is now inlinable because its /// in-callee guard resumes through a multi-frame snapshot /// ([`walker_capture_snapshot_for_last_guard_impl`]'s parent-frame branch). @@ -553,7 +553,7 @@ pub(crate) fn collect_callee_active_boxes( } /// #62: full-body-walk direct `CALL_ASSEMBLER` for a self-recursive call -/// at the inline recursion-bound boundary (dev-gated `PYRE_FBW_REC_CA`). +/// at the inline recursion-bound boundary. /// /// When the FBW inline depth for a callee reaches `FBW_MAX_INLINE_RECURSION` /// the call would otherwise degrade to a generic may-force residual, which @@ -597,13 +597,10 @@ pub(crate) fn try_walker_call_assembler_self_recursive( dst: usize, ) -> Result, DispatchError> { // ---- non-emitting eligibility checks (free to bail with Ok(None)) ---- - // Default ON since the Phase 5 flip; `PYRE_FBW_REC_CA=0` opts out. // Authoritative walks only: the CALL_ASSEMBLER record + walk-commit // bookkeeping is FBW machinery; a non-authoritative context (the // diagnostic probe, tests) records the plain residual instead. - if !ctx.is_authoritative_executor - || std::env::var_os("PYRE_FBW_REC_CA").as_deref() == Some(std::ffi::OsStr::new("0")) - { + if !ctx.is_authoritative_executor { return Ok(None); } // Only a genuine `call_fn` residual is a candidate — every @@ -720,8 +717,8 @@ pub(crate) fn try_walker_call_assembler_self_recursive( pyre_interpreter::live_code_wrapper((*sym.jitcode()).raw_code() as *const ()) as *const () }; // Self-fold requires callee code == portal code. The full-portal cutover - // (`PYRE_FBW_REC_MUTUAL_CUTOVER`) additionally admits a *mutual*-recursive - // callee — one whose code is already on the inline framestack, i.e. a + // additionally admits a *mutual*-recursive callee — one whose code is + // already on the inline framestack, i.e. a // genuine recursion cycle (`is_even` → `is_odd` → `is_even` at the unroll // cap). It must NOT admit an arbitrary foreign call: folding a // non-recursive callee (e.g. a CALL_KW-bearing leaf) to CALL_ASSEMBLER @@ -729,13 +726,12 @@ pub(crate) fn try_walker_call_assembler_self_recursive( // and faults. The emit below keys on `w_code` (callee-agnostic); the token // is resolved / synthesised per `callee_key` via `get_assembler_token`. if w_code as usize != caller_code as usize { - let admit_mutual = fbw_rec_mutual_cutover_enabled() - && ctx - .session - .borrow() - .framestack - .iter() - .any(|f| f.w_code == w_code as usize); + let admit_mutual = ctx + .session + .borrow() + .framestack + .iter() + .any(|f| f.w_code == w_code as usize); if !admit_mutual { return Ok(None); } @@ -777,14 +773,6 @@ pub(crate) fn try_walker_call_assembler_self_recursive( // carries a bodyless token. let (driver, _) = crate::driver::driver_pair(); let callee_key = crate::driver::make_green_key(w_code, 0); - let has_existing = driver.get_loop_token_arc(callee_key).is_some() - || driver.get_pending_token_arc(callee_key).is_some(); - if !has_existing && !fbw_rec_mutual_cutover_enabled() { - if std::env::var_os("PYRE_P2_DIAG").is_some() { - eprintln!("[p2-ca] decline pc={} reason=no-token", op.pc); - } - return Ok(None); - } // warmstate.py / compile.py: resolve an installed // procedure token, or synthesize a tmp callback token while the real loop // is still tracing. @@ -1054,7 +1042,7 @@ pub(crate) fn try_walker_call_assembler_self_recursive( /// at entry. The op sequence (vable/vref-before, CALL_ASSEMBLER + KEEPALIVE, /// residual executor to run the call concretely and stamp `ca_result`, dst /// writeback, GUARD_NOT_FORCED + GUARD_NO_EXCEPTION) mirrors -/// [`try_walker_call_assembler_self_recursive`]. `PYRE_FBW_LOOP_CALLEE_CA`. +/// [`try_walker_call_assembler_self_recursive`]. #[allow(clippy::too_many_arguments)] pub(crate) fn emit_walker_loop_callee_call_assembler( ctx: &mut WalkContext<'_, '_, Sym>, @@ -1135,8 +1123,7 @@ pub(crate) fn emit_walker_loop_callee_call_assembler( // binaries it does not reproduce across the GC-stress matrix // (r1/r5/r6/r2/r4 × nursery {default,1M,256K,64K,16K,4K} × dynasm+x86, all // clean) — a diagnostic-build layout artifact, with content-agnostic - // rooting ruling out a ref-specific defect here. See - // `fbw_loop_callee_ca_enabled` for the full default-ON rationale. + // rooting ruling out a ref-specific defect here. let argbox_types: Vec = vec![Type::Ref; r_args.len()]; let allboxes = build_allboxes(funcptr, r_args, &argbox_types, call_descr.arg_types()); let exec = try_execute_residual_call_via_executor( @@ -1204,12 +1191,11 @@ pub(crate) fn emit_loop_callee_ca_vable_scalar( } /// #62 slice (3c): full-body-walk inline of a recognized user-function -/// `call_fn`. Dev-gated by `PYRE_FBW_INLINE` (default OFF — the production -/// flag-on path is unchanged until this is validated and the gate retired). +/// `call_fn`. /// /// Returns: /// * `Ok(Some((outcome, next_pc)))` — the call was inlined; caller returns it. -/// * `Ok(None)` — not eligible (gate off, not a pure-Python function, has a +/// * `Ok(None)` — not eligible (not a pure-Python function, has a /// closure, or not an exact-positional call). This branch emits NO IR, so /// the caller's residual-call fallback is clean. /// * `Err(..)` — a sub-walk step hit an unsupported op AFTER emitting IR; @@ -1270,10 +1256,9 @@ pub(crate) fn try_walker_inline_user_call( dst_bank: char, dst: usize, ) -> Result, DispatchError> { - // Default ON since the Phase 5 flip; `PYRE_FBW_INLINE=0` opts out. // Authoritative walks only: inline sub-walks lean on FBW multi-frame // snapshot plumbing a non-authoritative context does not carry. - if !ctx.is_authoritative_executor || std::env::var("PYRE_FBW_INLINE").as_deref() == Ok("0") { + if !ctx.is_authoritative_executor { return Ok(None); } // Only a genuine Python call helper (`call_fn` / `call_fn_N`, tagged @@ -1432,14 +1417,14 @@ pub(crate) fn try_walker_inline_resolved_user_call( // The primary loop still inlines the callee, and non-integer/user- // overridable calls continue through the ordinary inline/abort policy. // - // Exception (`PYRE_FBW_BRIDGE_REC_INLINE`, default on): a plain ROOT bridge - // walk — no carrier resume, not an inline sub-walk, an empty framestack, and + // A plain ROOT bridge walk — no carrier resume, not an inline sub-walk, + // an empty framestack, and // a live root portal — is uniform with a primary trace, so its second // virtual frame is seeded and snapshot-covered exactly as the loop's is. // There the decline is lifted: the call falls through to the self-recursive // unroll gate and multiframe seed as if walked from a primary trace. - // True once this attempt takes the `PYRE_FBW_BRIDGE_REC_INLINE` root-bridge - // admission for a self-recursive callee. The admitted top-level inline's + // True once this attempt takes the root-bridge admission for a + // self-recursive callee. The admitted top-level inline's // body sub-walk reaches its own recursive CALL as a nested residual, which // `fbw_abort_nested_unjournaled_residual` declines on the self-recursive // hazard arm — an abort storm that folds the whole guard bridge back to @@ -1472,7 +1457,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( // continuation inlines instead of residualizing. let subwalk_admit = ctx.fbw_mode.carrier_resume && !ctx.fbw_mode.snapshot_sym.is_null(); let safe_root_bridge = root_bridge || subwalk_admit; - if !(fbw_bridge_rec_inline_enabled() && safe_root_bridge) { + if !safe_root_bridge { return Ok(None); } bridge_rec_root_selfrec = cfg!(not(target_arch = "wasm32")) @@ -1486,17 +1471,15 @@ pub(crate) fn try_walker_inline_resolved_user_call( // caller's CALL boundary, so deopt re-executes the whole callee. Replaying // a live-heap mutation would double it; the nested-residual decline catches // that only after an abort storm. A side-effect-free callee replays - // benignly, so admit it. `PYRE_FBW_FORITER_INLINE=0` restores the former - // blanket decline as a rollback escape hatch. + // benignly, so admit it. if fbw_foriter_inflight_active() - && (std::env::var("PYRE_FBW_FORITER_INLINE").as_deref() == Ok("0") - || !fbw_callee_body_side_effect_free( - body.code, - args_all_numeric, - body.num_regs_i, - body.constants_i, - callee_descr_refs, - )) + && !fbw_callee_body_side_effect_free( + body.code, + args_all_numeric, + body.num_regs_i, + body.constants_i, + callee_descr_refs, + ) { return Ok(None); } @@ -1567,11 +1550,8 @@ pub(crate) fn try_walker_inline_resolved_user_call( // callee is `try_multiframe`-eligible, but unbounded self-recursion bottoms // out the multiframe inline at the depth cap. A strict-inlinable callee is // a straight-line leaf (no self-recursion), so this never preempts the - // strict path. Gated on `PYRE_FBW_REC_CA`, matching the fold. - if !strict_inlinable - && std::env::var_os("PYRE_FBW_REC_CA").as_deref() != Some(std::ffi::OsStr::new("0")) - && nparams >= 1 - { + // strict path. + if !strict_inlinable && nparams >= 1 { let sym_ptr = ctx.fbw_mode.snapshot_sym; let self_recursive = !sym_ptr.is_null() && unsafe { @@ -1582,15 +1562,13 @@ pub(crate) fn try_walker_inline_resolved_user_call( if self_recursive { // RPython `opimpl_recursive_call` / `do_recursive_call` // (`pyjitpl.py`) unroll within `max_unroll_recursion`, - // then fall back to the assembler-call path. Default-on - // (`fbw_rec_multiframe_enabled`): a primary trace spends the + // then fall back to the assembler-call path. A primary trace spends the // recursion-unroll budget unrolling below `max_unroll_recursion` // (`fbw_max_rec_unroll_depth`, a bound distinct from the // straight-line chain-inline depth `fbw_max_multiframe_depth`) // before folding the deepest call to the recursive portal // `CALL_ASSEMBLER`. - let unroll = fbw_rec_multiframe_enabled() - && !ctx.fbw_mode.carrier_resume + let unroll = !ctx.fbw_mode.carrier_resume && ctx.session.borrow().framestack.len() < fbw_max_rec_unroll_depth(); if !unroll { return Ok(None); @@ -1598,8 +1576,8 @@ pub(crate) fn try_walker_inline_resolved_user_call( // fall through to the multiframe gate (unroll one level) } } - // #68: under `PYRE_FBW_INLINE_MULTIFRAME`, a forward-branch-bearing callee - // is inlinable with a multi-frame guard snapshot (its in-callee branch + // #68: a forward-branch-bearing callee is inlinable with a multi-frame + // guard snapshot (its in-callee branch // guard resumes through `walker_capture_multi_frame_inline_snapshot` rather // than collapsing to the caller boundary). The relaxed predicate also // accepts a callee whose only non-strict ops are reads off its OWN seeded @@ -1610,7 +1588,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( // intermediate callee jitcode) by `compute_inline_caller_frame`, bounded by // a depth cap on the inline stack (the `n_parents == n_callees` valve in // the snapshot path is the real desync safety net). - let multiframe_eligible = !strict_inlinable && fbw_inline_multiframe_enabled(); + let multiframe_eligible = !strict_inlinable; let callee_frame_reg = if multiframe_eligible { crate::state::ensure_jitcode_index(callee_code_key as *const ()) .map(|jc| crate::state::portal_red_regs_at(jc).0) @@ -1653,10 +1631,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( // the CALL_ASSEMBLER fold (`try_walker_call_assembler_self_recursive`, // reached next in the residual-call dispatch) so a recursive callee at // the inline cap enters via its own (possibly tmp-callback) loop token. - if fbw_rec_mutual_cutover_enabled() { - return Ok(None); - } - return Err(DispatchError::callee_inline_unsupported(op.pc)); + return Ok(None); } // Path-1 (#68): the inlined callee's compile-time-constant frame fields, @@ -1817,7 +1792,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( // an un-seedable strict shape never loses its inline. Every bail below // precedes any IR recording, so a strict fall-through records no dead op. // - // `PYRE_FBW_LOOP_CALLEE_CA`: the seeded virtual callee frame / + // The seeded virtual callee frame / // shared ec / local count are hoisted so the sub-walk return site can // emit a `CALL_ASSEMBLER` into the callee loop token when the sub-walk // surfaces `SubLoopCalleeCallAssembler` (the callee reached its own loop diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index f92abc642f1..8019720819b 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -1280,8 +1280,7 @@ pub enum DispatchOutcome { /// via `setarrayitem_vable` during the prologue walk; the caller sets /// `last_instr = target_pc - 1` on it and passes it as the /// CALL_ASSEMBLER `[frame, ec]` red arg (forcing the virtual - /// materializes the locals). Gated `PYRE_FBW_LOOP_CALLEE_CA` - /// (default-OFF); surfaced only from an inlined sub-walk. + /// materializes the locals); surfaced only from an inlined sub-walk. SubLoopCalleeCallAssembler { token: std::sync::Arc, target_pc: usize, @@ -1775,7 +1774,7 @@ pub enum DispatchError { /// rewind it and a deliver re-run would double it, so the walk declines BEFORE /// the commit and the location interprets permanently (`AbortPermanent`). InplaceContainerMutationUnsupported { pc: usize }, - /// Exception-edge bridge (`PYRE_EXC_EDGE_BRIDGE`): the failing exception + /// Exception-edge bridge: the failing exception /// guard is caught in-frame, but the `except` handler RETURNS out of the /// frame (a called function's `try/except: return`, compiled as its own /// function trace) rather than rejoining this frame's loop. Routing the @@ -2362,20 +2361,11 @@ pub(crate) fn try_catch_exception_at(code: &[u8], position: usize) -> Option bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| { - if cfg!(target_arch = "wasm32") { - return std::env::var_os("PYRE_EXC_EDGE_BRIDGE").is_some(); - } - match std::env::var_os("PYRE_EXC_EDGE_BRIDGE") { - Some(v) => v != "0", - None => true, - } - }) + cfg!(not(target_arch = "wasm32")) } /// `PYRE_CARRIER_EXC_RESUME=1` enables the multi-frame (carrier) exception @@ -3950,36 +3940,31 @@ fn collect_outer_active_boxes( }; let pcdep_opt: Option<&[(u8, u16, u16)]> = (!pcdep_entries.is_empty()).then(|| pcdep_entries.as_slice()); - let stack_livereg_gate = fbw_stack_livereg_enabled(); - let (guard_pcdep_entries, guard_stack_only) = if stack_livereg_gate { - if guard_present { - if sym.jitcode().is_null() { - (Vec::new(), 0usize) - } else { - unsafe { - let jc = &*sym.jitcode(); - let cjc = carried_jitcode_pc; - let entries = if cjc >= 0 { - jc.payload - .pcdep_for_jitcode_pc(cjc as usize) - .unwrap_or_default() - } else { - Vec::new() - }; - let depth = if jc.payload.code_ptr.is_null() { - 0usize - } else if cjc >= 0 { - jc.payload - .depth_for_jitcode_pc_pred(cjc as usize) - .unwrap_or(0) as usize - } else { - 0 - }; - (entries, depth) - } - } - } else { + let (guard_pcdep_entries, guard_stack_only) = if guard_present { + if sym.jitcode().is_null() { (Vec::new(), 0usize) + } else { + unsafe { + let jc = &*sym.jitcode(); + let cjc = carried_jitcode_pc; + let entries = if cjc >= 0 { + jc.payload + .pcdep_for_jitcode_pc(cjc as usize) + .unwrap_or_default() + } else { + Vec::new() + }; + let depth = if jc.payload.code_ptr.is_null() { + 0usize + } else if cjc >= 0 { + jc.payload + .depth_for_jitcode_pc_pred(cjc as usize) + .unwrap_or(0) as usize + } else { + 0 + }; + (entries, depth) + } } } else { (Vec::new(), 0usize) @@ -4180,14 +4165,11 @@ fn collect_outer_active_boxes( // stack slot at the guard capture point — there the // register read means exactly `registers_r[index]`; // where ownership is unprovable the virtualizable - // shadow remains authoritative - // (`PYRE_FBW_STACK_LIVEREG=0` restores shadow-first - // everywhere). + // shadow remains authoritative. let shadow_is_real = vbox.is_some_and(|b| !opref_is_null_const_ptr(b)); let walk_real = walk_box.filter(|&v| v != OpRef::NONE && !opref_is_null_const_ptr(v)); - let guard_pc_proves_slot = stack_livereg_gate - && guard_present + let guard_pc_proves_slot = guard_present && crate::state::semantic_ref_slot_for_reg_color( nlocals, guard_stack_only, @@ -4734,7 +4716,7 @@ thread_local! { static FBW_ABORT_CALL_RESUME: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; - /// B3 (`PYRE_FBW_RAISE`): the set of OpRefs the walker built inline via + /// B3: the set of OpRefs the walker built inline via /// [`try_walker_trace_exception_new`] (the virtualizable `NewWithVtable` /// exception). The immediately-following `RaiseVarargs` residual consults /// it to take the instance fast path — skipping the @@ -4745,7 +4727,7 @@ thread_local! { static FBW_BUILT_EXC: std::cell::RefCell> = std::cell::RefCell::new(std::collections::HashSet::new()); - /// B3 (`PYRE_FBW_RAISE`): LIFO stack of the previous-exception slot value + /// B3: LIFO stack of the previous-exception slot value /// saved by each lowered `PUSH_EXC_INFO` (`get_current_exception` arm), /// paired with its live concrete. `POP_EXCEPT`'s restore consumes the top /// entry, so the slot is set back to the TRUE saved prev (None for an outer @@ -7385,8 +7367,7 @@ fn guarded_branch_core( // slot to recover. Treat it as depth 0. // // Scope this to the collapse case ONLY: the #68 multiframe - // inline path (`PYRE_FBW_INLINE_MULTIFRAME`, - // `n_parents == n_callees`, both > 0) resumes the callee at its + // inline path (`n_parents == n_callees`, both > 0) resumes the callee at its // OWN pc through `GuardCaptureScope::branch_guard_jitcode_pc` // (`walker_capture_multi_frame_inline_snapshot`), so its // kept-stack branches still need the real depth/recovery. @@ -8899,28 +8880,23 @@ fn handle( } } if ctx.is_top_level { - if fbw_call_assembler_enabled() { - // Slice b: route the loop-free portal exit through - // `TraceAction::Finish` so the compile pipeline records - // the FINISH from `finish_args`. Re-box to Type::Ref + - // store_token_in_vable, then stash the payload; do NOT - // call `ctx.trace_ctx.finish()` here (would double-record). - // - // No-replay portal exit: also stash the CONCRETE return - // so `eval.rs` returns the walk's result directly instead - // of re-running the compiled trace against the already - // side-effected heap (the walk consumed it). A null / - // unknown concrete leaves the cell `None` → degrade. - if let ConcreteValue::Ref(ptr) = read_ref_reg_concrete(code, op, 0, ctx) { - if !ptr.is_null() { - fbw_finish_concrete_set(ConcreteValue::Ref(ptr)); - } + // Slice b: route the loop-free portal exit through + // `TraceAction::Finish` so the compile pipeline records + // the FINISH from `finish_args`. Re-box to Type::Ref + + // store_token_in_vable, then stash the payload; do NOT + // call `ctx.trace_ctx.finish()` here (would double-record). + // + // No-replay portal exit: also stash the CONCRETE return + // so `eval.rs` returns the walk's result directly instead + // of re-running the compiled trace against the already + // side-effected heap (the walk consumed it). A null / + // unknown concrete leaves the cell `None` → degrade. + if let ConcreteValue::Ref(ptr) = read_ref_reg_concrete(code, op, 0, ctx) { + if !ptr.is_null() { + fbw_finish_concrete_set(ConcreteValue::Ref(ptr)); } - fbw_terminate_with_finish(ctx, result, op.pc)?; - } else { - ctx.trace_ctx - .finish(&[result], ctx.done_with_this_frame_descr_ref.clone()); } + fbw_terminate_with_finish(ctx, result, op.pc)?; Ok((DispatchOutcome::Terminate, op.next_pc)) } else { Ok(( @@ -8949,22 +8925,17 @@ fn handle( } } if ctx.is_top_level { - if fbw_call_assembler_enabled() { - // Slice b: portal-exit FINISH carries Type::Ref even for - // an int return (the eval_loop_jit result_type is REF), - // so `fbw_ensure_boxed_for_ca` re-boxes via wrapint. - // - // No-replay portal exit: stash the concrete int so - // `eval.rs` returns the walk's result directly (re-boxed - // via `ConcreteValue::to_pyobj`). - if let ConcreteValue::Int(v) = read_int_reg_concrete(code, op, 0, ctx) { - fbw_finish_concrete_set(ConcreteValue::Int(v)); - } - fbw_terminate_with_finish(ctx, result, op.pc)?; - } else { - ctx.trace_ctx - .finish(&[result], ctx.done_with_this_frame_descr_int.clone()); + // Slice b: portal-exit FINISH carries Type::Ref even for + // an int return (the eval_loop_jit result_type is REF), + // so `fbw_ensure_boxed_for_ca` re-boxes via wrapint. + // + // No-replay portal exit: stash the concrete int so + // `eval.rs` returns the walk's result directly (re-boxed + // via `ConcreteValue::to_pyobj`). + if let ConcreteValue::Int(v) = read_int_reg_concrete(code, op, 0, ctx) { + fbw_finish_concrete_set(ConcreteValue::Int(v)); } + fbw_terminate_with_finish(ctx, result, op.pc)?; Ok((DispatchOutcome::Terminate, op.next_pc)) } else { Ok(( @@ -8985,13 +8956,8 @@ fn handle( let value = code[op.pc + 1] as i8 as i64; let result = OpRef::ConstInt(value); if ctx.is_top_level { - if fbw_call_assembler_enabled() { - fbw_finish_concrete_set(ConcreteValue::Int(value)); - fbw_terminate_with_finish(ctx, result, op.pc)?; - } else { - ctx.trace_ctx - .finish(&[result], ctx.done_with_this_frame_descr_int.clone()); - } + fbw_finish_concrete_set(ConcreteValue::Int(value)); + fbw_terminate_with_finish(ctx, result, op.pc)?; Ok((DispatchOutcome::Terminate, op.next_pc)) } else { Ok(( @@ -9013,18 +8979,13 @@ fn handle( // Operand layout `f`: 1B float register at op.pc+1. let result = read_float_reg(code, op, 0, ctx)?; if ctx.is_top_level { - if fbw_call_assembler_enabled() { - // Slice b: portal-exit FINISH carries Type::Ref; - // `fbw_ensure_boxed_for_ca` re-boxes the float via - // wrapfloat. - if let Some(majit_ir::Value::Float(v)) = ctx.trace_ctx.box_value(result) { - fbw_finish_concrete_set(ConcreteValue::Float(v)); - } - fbw_terminate_with_finish(ctx, result, op.pc)?; - } else { - ctx.trace_ctx - .finish(&[result], ctx.done_with_this_frame_descr_float.clone()); + // Slice b: portal-exit FINISH carries Type::Ref; + // `fbw_ensure_boxed_for_ca` re-boxes the float via + // wrapfloat. + if let Some(majit_ir::Value::Float(v)) = ctx.trace_ctx.box_value(result) { + fbw_finish_concrete_set(ConcreteValue::Float(v)); } + fbw_terminate_with_finish(ctx, result, op.pc)?; Ok((DispatchOutcome::Terminate, op.next_pc)) } else { Ok(( @@ -9052,25 +9013,20 @@ fn handle( // marker for void calls). // No operand bytes (the `/` argcodes is empty). if ctx.is_top_level { - if fbw_call_assembler_enabled() { - // Slice b: route the void portal exit through - // `TraceAction::Finish` (empty args) so the compile - // pipeline records the FINISH(void) from `finish_args`, - // mirroring the three value-returning arms. Store the - // assembler token in the vable + GUARD_NOT_FORCED_2 like - // those arms, then stash a void-marked payload; do NOT - // call `ctx.trace_ctx.finish()` here (would double-record). - // - // No-replay portal exit: stash `Null` (= void → None at - // the consume site) so a side-effecting void function - // returns directly instead of re-running its already - // applied effects. - fbw_finish_concrete_set(ConcreteValue::Null); - fbw_terminate_void_with_finish(ctx, op.pc)?; - } else { - ctx.trace_ctx - .finish(&[], ctx.done_with_this_frame_descr_void.clone()); - } + // Slice b: route the void portal exit through + // `TraceAction::Finish` (empty args) so the compile + // pipeline records the FINISH(void) from `finish_args`, + // mirroring the three value-returning arms. Store the + // assembler token in the vable + GUARD_NOT_FORCED_2 like + // those arms, then stash a void-marked payload; do NOT + // call `ctx.trace_ctx.finish()` here (would double-record). + // + // No-replay portal exit: stash `Null` (= void → None at + // the consume site) so a side-effecting void function + // returns directly instead of re-running its already + // applied effects. + fbw_finish_concrete_set(ConcreteValue::Null); + fbw_terminate_void_with_finish(ctx, op.pc)?; Ok((DispatchOutcome::Terminate, op.next_pc)) } else { Ok((DispatchOutcome::SubReturn { result: None }, op.next_pc)) @@ -9177,30 +9133,19 @@ fn handle( } } } - // Gated `PYRE_FBW_RAISE`: route the top-level raise through - // `SubRaise` so walk()'s SubRaise arm runs the in-frame + // Route the top-level raise through `SubRaise` so walk()'s + // SubRaise arm runs the in-frame // `catch_exception/L` lookahead (`finishframe_lookahead_at`) and // jumps into the handler instead of recording the top-level // exit-frame finish (which escapes a try/except as a no-payload // Terminate abort). - if ctx.is_top_level && !fbw_raise_enabled() { - ctx.trace_ctx - .finish(&[exc], ctx.exit_frame_with_exception_descr_ref.clone()); - if let ConcreteValue::Ref(p) = concrete_exc { - if !p.is_null() { - fbw_finish_raise_set(concrete_exc); - } - } - Ok((DispatchOutcome::Terminate, op.next_pc)) - } else { - Ok(( - DispatchOutcome::SubRaise { - exc, - exc_concrete: concrete_exc, - }, - op.next_pc, - )) - } + Ok(( + DispatchOutcome::SubRaise { + exc, + exc_concrete: concrete_exc, + }, + op.next_pc, + )) } "last_exc_value/>r" => { // RPython parity: `pyjitpl.py opimpl_last_exc_value`: @@ -9321,25 +9266,14 @@ fn handle( let exc = ctx .last_exc_value .ok_or(DispatchError::ReraiseWithoutLastExcValue { pc: op.pc })?; - // Gated `PYRE_FBW_RAISE`: symmetric with `raise/r`. - if ctx.is_top_level && !fbw_raise_enabled() { - ctx.trace_ctx - .finish(&[exc], ctx.exit_frame_with_exception_descr_ref.clone()); - if let ConcreteValue::Ref(p) = ctx.last_exc_value_concrete { - if !p.is_null() { - fbw_finish_raise_set(ctx.last_exc_value_concrete); - } - } - Ok((DispatchOutcome::Terminate, op.next_pc)) - } else { - Ok(( - DispatchOutcome::SubRaise { - exc, - exc_concrete: ctx.last_exc_value_concrete, - }, - op.next_pc, - )) - } + // Symmetric with `raise/r`. + Ok(( + DispatchOutcome::SubRaise { + exc, + exc_concrete: ctx.last_exc_value_concrete, + }, + op.next_pc, + )) } // The `i` spelling is what the assembler emits once the jitdriver // index outstrips a signed byte. Its leading byte names an Int-bank @@ -9394,7 +9328,7 @@ fn handle( Some(Value::Int(v)) => v as usize, _ => return Err(DispatchError::JitMergePointGreenKeyUnresolved { pc: op.pc }), }; - // `PYRE_FBW_LOOP_CALLEE_CA`: an inlined callee's own loop + // An inlined callee's own loop // header routes to a `CALL_ASSEMBLER` into its already-compiled loop // token EVEN WHEN its pycode green resolves. nbody's `advance` has a // const-Ref code_green (resolves) plus an existing loop token, so the @@ -9438,46 +9372,43 @@ fn handle( } } }; - if fbw_loop_callee_ca_enabled() { - let callee_code = ctx - .session - .borrow() - .framestack - .last() - .map(|frame| frame.w_code) - .filter(|&cc| { - fbw_root_code.is_none_or(|root| root as *const () != cc as *const ()) - }); - if let Some(callee_code) = callee_code { - let callee_key = - crate::driver::make_green_key(callee_code as *const (), next_instr); - let (driver, _) = crate::driver::driver_pair(); - let greenboxes = [ - Value::Int(next_instr as i64), - Value::Int(0), - Value::Ref(majit_ir::GcRef(callee_code)), - ]; - let red_types = [Type::Ref, Type::Ref]; - if let Some(token) = driver.get_or_make_portal_assembler_token_arc( - callee_key, - &greenboxes, - &red_types, - ) { - return Ok(( - DispatchOutcome::SubLoopCalleeCallAssembler { - token, - target_pc: next_instr, - }, - op.next_pc, - )); - } + let callee_code = ctx + .session + .borrow() + .framestack + .last() + .map(|frame| frame.w_code) + .filter(|&cc| { + fbw_root_code.is_none_or(|root| root as *const () != cc as *const ()) + }); + if let Some(callee_code) = callee_code { + let callee_key = + crate::driver::make_green_key(callee_code as *const (), next_instr); + let (driver, _) = crate::driver::driver_pair(); + let greenboxes = [ + Value::Int(next_instr as i64), + Value::Int(0), + Value::Ref(majit_ir::GcRef(callee_code)), + ]; + let red_types = [Type::Ref, Type::Ref]; + if let Some(token) = driver.get_or_make_portal_assembler_token_arc( + callee_key, + &greenboxes, + &red_types, + ) { + return Ok(( + DispatchOutcome::SubLoopCalleeCallAssembler { + token, + target_pc: next_instr, + }, + op.next_pc, + )); } } let code_ptr = match ctx.trace_ctx.concrete_of_opref(code_green) { Some(Value::Ref(gcref)) if gcref.0 != 0 => gcref.0 as *const (), _ => { - // `PYRE_FBW_LOOP_CALLEE_CA` (default-ON): inside a - // multi-frame inline sub-walk the callee's own + // Inside a multi-frame inline sub-walk the callee's own // `jit_merge_point` (its loop header) carries a pycode green // with no live Ref shadow, so this resolution fails and the // enclosing trace would decline. Recover @@ -9486,36 +9417,34 @@ fn handle( // surface a recursive CALL_ASSEMBLER request to the caller's // inline return site (mirror `opimpl_recursive_call_ // assembler`, metainterp.rs). - if fbw_loop_callee_ca_enabled() { - let callee_code = ctx - .session - .borrow() - .framestack - .last() - .map(|frame| frame.w_code); - if let Some(callee_code) = callee_code { - let callee_key = - crate::driver::make_green_key(callee_code as *const (), next_instr); - let (driver, _) = crate::driver::driver_pair(); - let greenboxes = [ - Value::Int(next_instr as i64), - Value::Int(0), - Value::Ref(majit_ir::GcRef(callee_code)), - ]; - let red_types = [Type::Ref, Type::Ref]; - if let Some(token) = driver.get_or_make_portal_assembler_token_arc( - callee_key, - &greenboxes, - &red_types, - ) { - return Ok(( - DispatchOutcome::SubLoopCalleeCallAssembler { - token, - target_pc: next_instr, - }, - op.next_pc, - )); - } + let callee_code = ctx + .session + .borrow() + .framestack + .last() + .map(|frame| frame.w_code); + if let Some(callee_code) = callee_code { + let callee_key = + crate::driver::make_green_key(callee_code as *const (), next_instr); + let (driver, _) = crate::driver::driver_pair(); + let greenboxes = [ + Value::Int(next_instr as i64), + Value::Int(0), + Value::Ref(majit_ir::GcRef(callee_code)), + ]; + let red_types = [Type::Ref, Type::Ref]; + if let Some(token) = driver.get_or_make_portal_assembler_token_arc( + callee_key, + &greenboxes, + &red_types, + ) { + return Ok(( + DispatchOutcome::SubLoopCalleeCallAssembler { + token, + target_pc: next_instr, + }, + op.next_pc, + )); } } top_level_live_code(ctx) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 8ee6c57ca19..4cb62c4b2ed 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -375,7 +375,7 @@ fn capture_escape_flush_undo(frame: usize) { /// Put the pre-flush frame state back. Called on every path that does not /// adopt the committed escape pc (commit withdrawal, an unforced or -/// rootless continuation, the `PYRE_FBW_ABORT_FLUSH=0` opt-out) so the +/// rootless continuation) so the /// legacy replay re-enters a pristine frame. pub(crate) fn restore_escape_flush_undo() { ESCAPE_FLUSH_UNDO.with(|slot| { @@ -837,10 +837,10 @@ pub(crate) fn walker_abort_if_mayforce_null_ref_arg( let is_call_fn = call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::CallFn; // `RaiseVarargs` (`normalize_raise_varargs`) carries a trailing `cause` // Ref that is a checked `PY_NULL` sentinel for `raise X` without `from` - // (never dereferenced when null); exempt it (gated `PYRE_FBW_RAISE`) so the + // (never dereferenced when null); exempt it so the // FBW path can own the raise instead of declining to the trait. - let is_raise_varargs = fbw_raise_enabled() - && call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::RaiseVarargs; + let is_raise_varargs = + call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::RaiseVarargs; // `bh_call_function_ex_fn(callable, self_or_null, starargs, kwargs_or_null)` // — `self_or_null` (arg 1) and `kwargs_or_null` (arg 3) are checked // `PY_NULL` sentinels (never dereferenced when null), so a concrete-NULL @@ -1242,9 +1242,9 @@ pub(crate) fn try_execute_residual_call_via_executor( let is_call_function_ex = call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::CallFunctionEx; // Same `RaiseVarargs` trailing-`cause` sentinel exemption as - // `walker_abort_if_mayforce_null_ref_arg` (gated `PYRE_FBW_RAISE`). - let is_raise_varargs = fbw_raise_enabled() - && call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::RaiseVarargs; + // `walker_abort_if_mayforce_null_ref_arg`. + let is_raise_varargs = + call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::RaiseVarargs; for (i, &arg) in args.iter().enumerate() { if is_call_fn && i == 1 { continue; @@ -1696,7 +1696,7 @@ pub(crate) fn try_execute_residual_call_via_executor( } // On a kept commit the undo stays armed: the abort epilogue // consumes it — discard on adoption, restore when the flush is - // not adopted (`PYRE_FBW_ABORT_FLUSH=0`). + // not adopted. // On the cancel arm the restore above ran FIRST, so this refresh // reloads PRE-walk values — the shadow then matches the frame the // legacy replay will use, not walk-end state. A future ladder @@ -2286,7 +2286,7 @@ pub(crate) fn dispatch_residual_call_iRd_kind( clear_walk_exception(ctx); // #62 slice (3c): attempt full-body-walk inline of a user-function call - // (dev-gated PYRE_FBW_INLINE). Eligible exact-positional closure-free + // unconditionally. Eligible exact-positional closure-free // calls sub-walk the callee body in place of the residual; ineligible // calls (including every non-`call_fn` helper, gated on `pyre_helper`) // fall through with no IR emitted. @@ -2318,7 +2318,7 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // #62: a self-recursive call the inline path declined (e.g. the // branchy `fib`) gets a direct `CALL_ASSEMBLER` to its own loop token - // (dev-gated PYRE_FBW_REC_CA) instead of the heavyweight func-entry + // instead of the heavyweight func-entry // residency residual. Independent of inline eligibility. if let Some(ca) = try_walker_call_assembler_self_recursive( ctx, @@ -2427,8 +2427,8 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // handler-free gate as the LoadName fold (the fold elides a can-raise // residual a `catch_exception/L` could resume into). // - // Default ON (`PYRE_FBW_STORENAME_FOLD=0` opts out). Two staleness bugs - // fixed before the flip: (1) the fold now eagerly applies the concrete + // Two staleness bugs were fixed before enabling this unconditionally: + // (1) the fold now eagerly applies the concrete // `cell.intvalue` write (journaled in [`FBW_CELL_STORE_JOURNAL`]) — // without it the walk's remaining concrete execution read the pre-store // global and the next LOAD fold's cache-hit sanity check tripped; @@ -2447,7 +2447,6 @@ pub(crate) fn dispatch_residual_call_iRd_kind( majit_ir::PyreHelperKind::StoreName | majit_ir::PyreHelperKind::StoreGlobal ) && !jitcode_has_exception_handler(code) - && std::env::var("PYRE_FBW_STORENAME_FOLD").as_deref() != Ok("0") { if let (Some(&frame_opref), Some(&name_opref), Some(&value_opref)) = (r_args.first(), r_args.get(1), r_args.get(2)) @@ -2693,7 +2692,7 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } - // B3 (`PYRE_FBW_RAISE`, default OFF): a `raise Type(args)` of a canonical + // B3: a `raise Type(args)` of a canonical // builtin exception class arrives as two residuals — a `CallFn` that // constructs the exception, and a `RaiseVarargs` // (`normalize_raise_varargs_jit`) that publishes it. The construct fold @@ -2707,7 +2706,6 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // the exception virtualizes and DCEs. Any non-matching shape falls // through to the generic residual (SAFE). if ctx.is_authoritative_executor - && fbw_raise_enabled() && dst_bank == 'r' && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn && try_walker_trace_exception_new(ctx, code, op, &r_args, dst)?.is_some() @@ -2715,7 +2713,6 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } if ctx.is_authoritative_executor - && fbw_raise_enabled() && dst_bank == 'r' && ei.pyre_helper == majit_ir::PyreHelperKind::RaiseVarargs && r_args.is_empty() @@ -2738,14 +2735,13 @@ pub(crate) fn dispatch_residual_call_iRd_kind( } } if ctx.is_authoritative_executor - && fbw_raise_enabled() && dst_bank == 'r' && ei.pyre_helper == majit_ir::PyreHelperKind::RaiseVarargs && try_walker_trace_raise_builtin(ctx, code, op, &r_args, dst)?.is_some() { return Ok((DispatchOutcome::Continue, op.next_pc)); } - // B3 piece 3 (`PYRE_FBW_RAISE`): lower the PUSH_EXC_INFO / POP_EXCEPT + // B3 piece 3: lower the PUSH_EXC_INFO / POP_EXCEPT // exc-info-stack residuals to GETFIELD_GC_R / SETFIELD_GC on the EC's // `sys_exc_value` slot. Recognised by the // codewriter-stamped `pyre_helper` tag (not a funcptr address — the @@ -2756,7 +2752,6 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // virtual and DCEs — eliding the per-iteration `set_current_exception` // CALL that otherwise forces the exception to materialize. if ctx.is_authoritative_executor - && fbw_raise_enabled() && matches!( ei.pyre_helper, majit_ir::PyreHelperKind::GetCurrentException @@ -3082,7 +3077,6 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // values retain the original CallMayForceN unchanged. if ctx.is_authoritative_executor && original_call_descr.get_extra_info().pyre_helper == majit_ir::PyreHelperKind::StoreAttr - && fbw_storeattr_fold_enabled() { if let (Some(&obj_opref), Some(&value_opref), Some(&code_opref), Some(&namei_opref)) = (r_args.first(), r_args.get(1), r_args.get(2), i_args.first()) @@ -3219,16 +3213,13 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // for this load is sound — the handler can never be entered from it. We // therefore attempt the fold even in handler-bearing bodies and keep the // residual (with its guard) only when the fold DECLINES. The `B3`/builtin - // raise+catch path (`PYRE_FBW_BUILTIN_FOLD`) needs this so the + // raise+catch path needs this so the // `raise ValueError`/`except ValueError` class loads fold to const. // - // Default ON since the Phase 5 flip (`PYRE_FBW_LOADGLOBAL_FOLD=0` opts - // out): the fold is correct (`try_walker_load_global_cell_fold` - // resolves the `co_names` index the same way `bh_load_global_fn` does) - // and reaches production parity for global-function-call loops when - // combined with the user-call inlining path. The handler-bearing - // reachability is additionally gated `PYRE_FBW_BUILTIN_FOLD` (default ON) - // so the legacy handler-free behavior is recoverable. + // The fold resolves the `co_names` index the same way + // `bh_load_global_fn` does and reaches production parity for + // global-function-call loops when combined with the user-call inlining + // path. Handler-bearing reachability also includes the builtins fallback. if ctx.is_authoritative_executor && ei.pyre_helper == majit_ir::PyreHelperKind::LoadGlobal { if let (Some(&namei_opref), Some(&ns_opref), Some(&code_opref)) = (i_args.first(), r_args.first(), r_args.get(1)) @@ -3258,11 +3249,7 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( ctx.trace_ctx.reads_module_global = true; } } - if ctx.is_authoritative_executor - && ei.pyre_helper == majit_ir::PyreHelperKind::LoadGlobal - && std::env::var("PYRE_FBW_LOADGLOBAL_FOLD").as_deref() != Ok("0") - && (!jitcode_has_exception_handler(code) || fbw_builtin_fold_enabled()) - { + if ctx.is_authoritative_executor && ei.pyre_helper == majit_ir::PyreHelperKind::LoadGlobal { if let (Some(&namei_opref), Some(&ns_opref), Some(&code_opref)) = (i_args.first(), r_args.first(), r_args.get(1)) { @@ -3328,7 +3315,6 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( if ctx.is_authoritative_executor && ei.pyre_helper == majit_ir::PyreHelperKind::LoadName && !jitcode_has_exception_handler(code) - && std::env::var("PYRE_FBW_LOADNAME_FOLD").as_deref() != Ok("0") { if let (Some(&frame_opref), Some(&name_opref)) = (r_args.first(), r_args.get(1)) { if let ( @@ -3355,10 +3341,7 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // guard proves the attribute is present on this shape), so it is attempted // even in handler-bearing bodies; every unfoldable shape falls through to // the residual (which keeps its exception guard). - if ctx.is_authoritative_executor - && ei.pyre_helper == majit_ir::PyreHelperKind::LoadAttr - && fbw_loadattr_fold_enabled() - { + if ctx.is_authoritative_executor && ei.pyre_helper == majit_ir::PyreHelperKind::LoadAttr { if let (Some(&obj_opref), Some(&code_opref), Some(&namei_opref)) = (r_args.first(), r_args.get(1), i_args.first()) { @@ -3387,7 +3370,6 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( } if ctx.is_authoritative_executor && ei.pyre_helper == majit_ir::PyreHelperKind::LoadAttr - && fbw_loadmethod_fold_enabled() && next_op_is_load_method_self_for_attr(code, op, ctx, dst) { if let (Some(&obj_opref), Some(&code_opref), Some(&namei_opref)) = @@ -3416,10 +3398,7 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( } } } - if ctx.is_authoritative_executor - && ei.pyre_helper == majit_ir::PyreHelperKind::LoadMethodSelf - && fbw_loadmethod_fold_enabled() - { + if ctx.is_authoritative_executor && ei.pyre_helper == majit_ir::PyreHelperKind::LoadMethodSelf { if let (Some(&namei_opref), Some(&obj_opref), Some(&attr_opref), Some(&code_opref)) = (i_args.first(), r_args.first(), r_args.get(1), r_args.get(2)) { @@ -3544,8 +3523,8 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( }, } } - } else if op_tag == 10 && fbw_raise_enabled() && ctx.is_authoritative_executor { - // B3 (`PYRE_FBW_RAISE`): `op_tag == 10` is CHECK_EXC_MATCH + } else if op_tag == 10 && ctx.is_authoritative_executor { + // B3: `op_tag == 10` is CHECK_EXC_MATCH // (`bh_compare_fn(exc, match_type, 10)`, // `call_jit.rs`). Fold the match concretely to a // const bool (the immortal TRUE/FALSE singleton) so the diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index 9f71abe1cd2..bb54441632e 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -131,33 +131,31 @@ pub(crate) fn walker_capture_inline_nonstandard_vable_guard( // the promote. A chain that is not full, or a callee/caller frame the // publisher cannot build, falls through to (or aborts the same as) the // sentinel below — never a wrong resume. - if fbw_nsvable_multiframe_enabled() { - let (n_parents, n_callees, parent_frames) = { - let session = ctx.session.borrow(); - ( - session - .framestack - .iter() - .filter(|frame| frame.parent.is_some()) - .count(), - session.framestack.len(), - session - .framestack - .iter() - .filter_map(|frame| frame.parent.clone()) - .collect::>(), - ) - }; - if n_parents > 0 && n_parents == n_callees { - return walker_capture_multi_frame_inline_snapshot( - ctx, - op_pc, - false, - parent_frames, - GuardCaptureScope::default(), - true, - ); - } + let (n_parents, n_callees, parent_frames) = { + let session = ctx.session.borrow(); + ( + session + .framestack + .iter() + .filter(|frame| frame.parent.is_some()) + .count(), + session.framestack.len(), + session + .framestack + .iter() + .filter_map(|frame| frame.parent.clone()) + .collect::>(), + ) + }; + if n_parents > 0 && n_parents == n_callees { + return walker_capture_multi_frame_inline_snapshot( + ctx, + op_pc, + false, + parent_frames, + GuardCaptureScope::default(), + true, + ); } // The guard is not the last recorded op: `emit_force_virtualizable` // records GETFIELD_GC / PTR_NE / COND_CALL after the promote, so stamp @@ -277,7 +275,7 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // sub-walk with paused caller frames on the walk framestack resumes // BOTH the callee (at its own pc) and the caller(s) (at the CALL return // point), instead of collapsing to the caller boundary (re-execute). Only - // the gated forward-branch inline path (`PYRE_FBW_INLINE_MULTIFRAME`) + // the forward-branch inline path // populates the chain; straight-line callees keep the empty chain + the // single-frame collapse below. if inline_subwalk { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index bf6e5784a18..6b595c7369d 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -2167,7 +2167,7 @@ pub(crate) fn try_walker_specialize_compare_op_int( Ok(Some(())) } -/// B3 (`PYRE_FBW_RAISE`): walker-native fold of the CHECK_EXC_MATCH +/// B3: walker-native fold of the CHECK_EXC_MATCH /// residual (`bh_compare_fn(exc, match_type, op_tag=10)`, /// `call_jit.rs`). Computes the match concretely from /// `type(exc)` and `match_type` and emit a `const_ref` of the immortal @@ -3843,7 +3843,7 @@ pub(crate) fn try_walker_orthodox_list_append_opcode( Ok(Some(())) } -/// B3 (`PYRE_FBW_RAISE`): walker-native exception-construction fold. A +/// B3: walker-native exception-construction fold. A /// `Type(args)` `CallFn` residual for a canonical builtin exception class or /// a heap subclass with the same `__new__` / `__init__` descriptors becomes a /// traced `NewWithVtable` + `SetfieldGc` (kind / w_class / args_w) the @@ -4284,7 +4284,7 @@ pub(crate) fn try_walker_trace_exception_new( Ok(Some(())) } -/// B3 (`PYRE_FBW_RAISE`): walker-native RAISE_VARARGS E1 fast path. The `RaiseVarargs` +/// B3: walker-native RAISE_VARARGS E1 fast path. The `RaiseVarargs` /// residual is `normalize_raise_varargs_jit(frame, exc, cause)` — /// `r_args = [frame, exc, cause]`. When `exc` was built inline by /// [`try_walker_trace_exception_new`] (∈ [`FBW_BUILT_EXC`]) and there is @@ -4418,7 +4418,7 @@ pub(crate) fn try_walker_trace_raise_builtin( Ok(Some(())) } -/// B3 piece 3 (`PYRE_FBW_RAISE`): lower the PUSH_EXC_INFO / POP_EXCEPT +/// B3 piece 3: lower the PUSH_EXC_INFO / POP_EXCEPT /// exc-info-stack residuals to GETFIELD_GC_R / SETFIELD_GC on the EC's /// `sys_exc_value` slot (`ec_sys_exc_value_descr`). /// Recognised by the codewriter-stamped `pyre_helper` tag, NOT a funcptr @@ -5517,13 +5517,12 @@ pub(crate) fn try_walker_specialize_compare_op_float( /// `Ok(false)` when the receiver is not a foldable cell (the caller then /// falls through to the generic residual, which stays correct). /// -/// DEV-GATED + INCOMPLETE: callers gate this on `PYRE_FBW_LOADGLOBAL_FOLD` -/// (default off). When the loaded global is a function that is then CALLed, -/// folding it to a loop-invariant constant callee routes the call through the -/// in-progress FBW call-inlining path (#68), which mis-resolves the callee and -/// produces wrong output. Keep default-off until #68 lands. +/// Callers fall back to the residual call when this fold declines. When the +/// loaded global is a function that is then CALLed, folding it to a +/// loop-invariant constant callee routes the call through the FBW call-inlining +/// path (#68). /// -/// Builtins fallback (`PYRE_FBW_BUILTIN_FOLD`): when `name` is ABSENT from the +/// Builtins fallback: when `name` is ABSENT from the /// module dict but resolves through `frame.get_builtin()` (e.g. /// `raise ValueError` / `except ValueError`), the same cell fold is emitted /// against the BUILTINS dict, guarded additionally by a `QUASIIMMUT_FIELD` on @@ -5565,13 +5564,10 @@ pub(crate) fn try_walker_load_global_cell_fold( return Ok(false); } - // Builtins fallback (`PYRE_FBW_BUILTIN_FOLD`): the name is absent from the + // Builtins fallback: the name is absent from the // `ns_ptr` module dict. Mirror `bh_load_global_fn`'s second leg — // `frame.get_builtin().getdictvalue(name)` — and fold the builtins cell // when the name resolves there. Requires the live frame operand. - if !fbw_builtin_fold_enabled() { - return Ok(false); - } // The builtins fallback needs the module `pick_builtin(w_globals)` picks // (`frame.get_builtin()`). A live frame supplies it directly and also lets // us double-check the name is absent from the frame's AUTHORITATIVE globals diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs index 9737fd0c208..71ac42b1264 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs @@ -3153,7 +3153,7 @@ fn step_through_void_return_stashes_void_finish_payload() { // `pyjitpl.py compile_done_with_this_frame`, the // `result_type == VOID` branch — `exits = []`, // `token = sd.done_with_this_frame_descr_void`). Under the - // `PYRE_FBW_CALL_ASSEMBLER` gate (default on) it mirrors the three + // finish-portal route it mirrors the three // value-returning arms: it does NOT record the FINISH op itself // (the compile consumer records `FINISH([])` from the empty // finish_args) and stashes a `Type::Void`-marked payload so @@ -3655,7 +3655,7 @@ fn step_through_raise_records_outermost_finish_and_terminates() { // `finishframe_exception` (outermost-frame branch) → // `compile_exit_frame_with_exception` records // `FINISH(exc, descr=exit_frame_with_exception_descr_ref)`. - // With `PYRE_FBW_RAISE` on (default), `raise/r` surfaces + // `raise/r` surfaces // `SubRaise` and `walk()`'s top-level SubRaise arm records the // outermost FINISH + converts to Terminate, so drive `walk()`. let raise_byte = *insns_opname_to_byte() @@ -3916,7 +3916,7 @@ fn step_through_reraise_at_top_level_records_outermost_finish() { live_before_jit_pc: usize::MAX, live_after_jit_pc: usize::MAX, }; - // With `PYRE_FBW_RAISE` on (default), `reraise/` surfaces + // `reraise/` surfaces // `SubRaise` and `walk()`'s top-level SubRaise arm records the // outermost FINISH + converts to Terminate. fbw_finish_payload_reset(); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/vable_ops.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/vable_ops.rs index ff81da3bbb8..d3335108a34 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/vable_ops.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/vable_ops.rs @@ -52,10 +52,8 @@ pub(crate) fn getfield_vable_via_metainterp( // descr-identity-independent (`const_ref`) and only intercepts the two // static Ref fields (`try_resolve_inline_callee_static_field` returns `None` // for everything else), so it is safe to consult on the seeded path too. - if fbw_inline_nsfold_enabled() { - if let Some(resolved) = try_resolve_inline_callee_static_field(code, op, ctx, dst_bank)? { - return Ok(resolved); - } + if let Some(resolved) = try_resolve_inline_callee_static_field(code, op, ctx, dst_bank)? { + return Ok(resolved); } // RPython's `box` is always a live virtualizable-frame box. An // unseeded walker Ref register holds `OpRef::None` (`raw() == diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index c346a3e9e35..fc6539567dd 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -8932,7 +8932,6 @@ impl JitState for PyreJitState { fail_values: &[i64], fail_types: &[Type], ) { - let bridge_stamp_enabled = std::env::var("PYRE_FBW_BRIDGE_STAMP").as_deref() != Ok("0"); if resume_data.frames.is_empty() { return; } @@ -9130,11 +9129,7 @@ impl JitState for PyreJitState { // GuardTrue/GuardFalse — orthodox meta-tracing ("trace the // concrete path, guard it"; the IR keeps the symbolic InputArg, // the concrete is a trace-time shadow only, so the optimizer does - // NOT const-fold the loop-variant value). Default-ON (`=0` opts - // out) once validated, mirroring the LoadGlobal-fold / multiframe - // flips; the opt-out keeps the prior symbolic-bridge - // behavior available for A/B. - let seed_bridge_locals = std::env::var("PYRE_FBW_BRIDGE_LOCAL_SEED").as_deref() != Ok("0"); + // NOT const-fold the loop-variant value). // `seed_deferred_to_overlay` is Ref-specific. The Ref overlay below // rebuilds a SLOT-indexed mirror from the color-indexed resume decode // (via `pcdep_entries`). At a kept-stack branch guard the body-internal @@ -9151,8 +9146,7 @@ impl JitState for PyreJitState { // uniformly at the guard's resume position. let seed_deferred_to_overlay = crate::state::frame_pc_is_resolved_offset_at(frame0.jitcode_index, frame0.pc); - let mut bridge_stamp_orphans = - (bridge_stamp_enabled && seed_deferred_to_overlay).then(Vec::new); + let mut bridge_stamp_orphans = seed_deferred_to_overlay.then(Vec::new); let mut value_cursor = 0usize; for ®_idx in ®_indices.int { let value = &frame0.values[value_cursor]; @@ -9176,7 +9170,7 @@ impl JitState for PyreJitState { // instead of declining the bridge with `GotoIfNotValueNotConcrete`. // Mirrors `consume_boxes(..., f.registers_i, ...)` filling the Int // bank at the guard's resume position. - if seed_bridge_locals && !matches!(concrete_val, majit_ir::Value::Void) { + if !matches!(concrete_val, majit_ir::Value::Void) { ctx.try_set_opref_concrete(resolved, concrete_val); } let reg_idx = reg_idx as usize; @@ -9199,10 +9193,7 @@ impl JitState for PyreJitState { backend, &mut virtuals_cache, ); - if seed_bridge_locals - && !seed_deferred_to_overlay - && !matches!(concrete_val, majit_ir::Value::Void) - { + if !seed_deferred_to_overlay && !matches!(concrete_val, majit_ir::Value::Void) { ctx.try_set_opref_concrete(resolved, concrete_val); } if let Some(orphan_stamps) = bridge_stamp_orphans.as_mut() { @@ -9232,7 +9223,7 @@ impl JitState for PyreJitState { // bank above; stamp its concrete unconditionally (the Ref-only // `seed_deferred_to_overlay` deferral does not apply), mirroring // `consume_boxes(..., f.registers_f)`. - if seed_bridge_locals && !matches!(concrete_val, majit_ir::Value::Void) { + if !matches!(concrete_val, majit_ir::Value::Void) { ctx.try_set_opref_concrete(resolved, concrete_val); } let reg_idx = reg_idx as usize; @@ -9327,15 +9318,12 @@ impl JitState for PyreJitState { // `MayForceNullRefArgUnsupported`. Leaving the box // unstamped keeps it symbolic so the residual reads // the runtime value. - if seed_bridge_locals { - if let Some(&cv) = live_local_values.get(s) { - if !matches!( - cv, - majit_ir::Value::Void - | majit_ir::Value::Ref(majit_ir::GcRef(0)) - ) { - ctx.try_set_opref_concrete(v, cv); - } + if let Some(&cv) = live_local_values.get(s) { + if !matches!( + cv, + majit_ir::Value::Void | majit_ir::Value::Ref(majit_ir::GcRef(0)) + ) { + ctx.try_set_opref_concrete(v, cv); } } } else if slot.is_none() { @@ -9446,11 +9434,9 @@ impl JitState for PyreJitState { if let Some(v) = vable_array_items.get(s).copied() { if !v.is_none() { mirror[s] = v; - if seed_bridge_locals { - if let Some(&cv) = live_local_values.get(s) { - if !matches!(cv, majit_ir::Value::Void) { - ctx.try_set_opref_concrete(v, cv); - } + if let Some(&cv) = live_local_values.get(s) { + if !matches!(cv, majit_ir::Value::Void) { + ctx.try_set_opref_concrete(v, cv); } } } @@ -9511,7 +9497,7 @@ impl JitState for PyreJitState { // slots — seeding there stamps the wrong value. After the // overlay the mirror is slot-indexed and authoritative, so seed // each non-NONE slot from the GC-rooted live frame values. - if seed_bridge_locals && seed_deferred_to_overlay { + if seed_deferred_to_overlay { for (s, opref) in semantic_mirror.iter().enumerate() { if !opref.is_none() { if let Some(&cv) = live_local_values.get(s) { @@ -9568,19 +9554,17 @@ impl JitState for PyreJitState { // the residual sees the real (null) sentinel and executes. Scoped to // operand slots (`>= nlocals`); locals keep the frame-array overlay, // and a slot already carrying a real concrete is left untouched. - if seed_bridge_locals { - if let Some(orphan_stamps) = bridge_stamp_orphans.as_ref() { - for s in nlocals..semantic_prefix_len { - let Some(&opref) = semantic_mirror.get(s) else { - continue; - }; - if opref.is_none() || ctx.box_value(opref).is_some() { - continue; - } - if let Some((_, cv)) = orphan_stamps.iter().find(|(o, _)| o == &opref) { - if !matches!(cv, majit_ir::Value::Void) { - ctx.try_set_opref_concrete(opref, *cv); - } + if let Some(orphan_stamps) = bridge_stamp_orphans.as_ref() { + for s in nlocals..semantic_prefix_len { + let Some(&opref) = semantic_mirror.get(s) else { + continue; + }; + if opref.is_none() || ctx.box_value(opref).is_some() { + continue; + } + if let Some((_, cv)) = orphan_stamps.iter().find(|(o, _)| o == &opref) { + if !matches!(cv, majit_ir::Value::Void) { + ctx.try_set_opref_concrete(opref, *cv); } } } diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 86e7e3b8428..a9b85b43cc1 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -2372,44 +2372,37 @@ fn run_perfn_walk( // state instead of replaying. The store-journal epilogue below // settles the walk's eager list stores either way (commit keeps // them, non-commit rolls them back for the replay). - // `PYRE_FBW_END_FLUSH=0` opts out for bisection. - if std::env::var_os("PYRE_FBW_END_FLUSH").as_deref() != Some(std::ffi::OsStr::new("0")) { - if let Ok((outcome, _end_pc)) = &walk_result { - let header_pc = match outcome { - crate::jitcode_dispatch::DispatchOutcome::CloseLoop { .. } => { - close_loop_restart_pc + if let Ok((outcome, _end_pc)) = &walk_result { + let header_pc = match outcome { + crate::jitcode_dispatch::DispatchOutcome::CloseLoop { .. } => close_loop_restart_pc, + crate::jitcode_dispatch::DispatchOutcome::CompileTracePending { + loop_header_pc, + } => Some(*loop_header_pc), + _ => None, + }; + if let Some(header_pc) = header_pc { + if crate::jitcode_dispatch::fbw_has_unjournaled_effect() { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-end-flush] declined at header_pc={header_pc} \ + (unjournaled effect) — legacy replay kept" + ); } - crate::jitcode_dispatch::DispatchOutcome::CompileTracePending { - loop_header_pc, - } => Some(*loop_header_pc), - _ => None, - }; - if let Some(header_pc) = header_pc { - if crate::jitcode_dispatch::fbw_has_unjournaled_effect() { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-end-flush] declined at header_pc={header_pc} \ - (unjournaled effect) — legacy replay kept" - ); - } - } else if crate::state::flush_walk_loop_end_state_to_frame( - ctx, cf_addr, header_pc, - ) { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-end-flush] COMMIT header_pc={header_pc} bridge={} \ - journal_len={} outcome={outcome:?}", - ctx.is_bridge_trace, - crate::jitcode_dispatch::fbw_store_journal_len(), - ); - } - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + } else if crate::state::flush_walk_loop_end_state_to_frame(ctx, cf_addr, header_pc) { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( - "[fbw-end-flush] declined at header_pc={header_pc} (shadow slot \ - without concrete / depth / lastblock) — legacy replay kept" + "[fbw-end-flush] COMMIT header_pc={header_pc} bridge={} \ + journal_len={} outcome={outcome:?}", + ctx.is_bridge_trace, + crate::jitcode_dispatch::fbw_store_journal_len(), ); } + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-end-flush] declined at header_pc={header_pc} (shadow slot \ + without concrete / depth / lastblock) — legacy replay kept" + ); } } } @@ -2429,287 +2422,260 @@ fn run_perfn_walk( // The marker-only fallback uses the same no-unjournaled-effect // predicate as the CloseLoop end-flush above. A latched inline-callee // forward abort has already distinguished an outside mark from a mark - // inside its discarded attempt. `PYRE_FBW_ABORT_FLUSH=0` opts out. + // inside its discarded attempt. let force_blackhole_adopted = matches!( &walk_result, Err(crate::jitcode_dispatch::DispatchError::VableEscapedDuringResidualCall { .. }) ) && try_adopt_force_blackhole(ctx, cf_addr); - if std::env::var_os("PYRE_FBW_ABORT_FLUSH").as_deref() == Some(std::ffi::OsStr::new("0")) { - // Opt-out: never adopt the escape flush — drop the commit and put - // the pre-flush frame back so the legacy replay sees pristine - // state. - if !force_blackhole_adopted - && matches!( - &walk_result, - Err( - crate::jitcode_dispatch::DispatchError::VableEscapedDuringResidualCall { .. } - ) - ) - { - let _ = crate::jitcode_dispatch::take_committed_frame_escape_pc(); - crate::jitcode_dispatch::restore_escape_flush_undo(); - } - } else { - if !force_blackhole_adopted - && matches!( - &walk_result, - Err( - crate::jitcode_dispatch::DispatchError::VableEscapedDuringResidualCall { .. } - ) - ) - && let Some(resume_py_pc) = - crate::jitcode_dispatch::take_committed_frame_escape_pc() - { - crate::jitcode_dispatch::discard_escape_flush_undo(); - // The force-time escape flush wrote the resume state into the - // LIVE frame (the frame the callee inspected). The portal - // epilogue propagates `executed_frame` → live on a committed - // flush, so mirror the live frame's resume state into the walk - // snapshot to make that copy the identity. - let live = sym.live_vable_frame_addr(); - if live != 0 && cf_addr != 0 && live != cf_addr { - unsafe { - (*(cf_addr as *mut pyre_interpreter::PyFrame)).restore_resume_state_from( - &*(live as *const pyre_interpreter::PyFrame), - ); - } + if !force_blackhole_adopted + && matches!( + &walk_result, + Err(crate::jitcode_dispatch::DispatchError::VableEscapedDuringResidualCall { .. }) + ) + && let Some(resume_py_pc) = crate::jitcode_dispatch::take_committed_frame_escape_pc() + { + crate::jitcode_dispatch::discard_escape_flush_undo(); + // The force-time escape flush wrote the resume state into the + // LIVE frame (the frame the callee inspected). The portal + // epilogue propagates `executed_frame` → live on a committed + // flush, so mirror the live frame's resume state into the walk + // snapshot to make that copy the identity. + let live = sym.live_vable_frame_addr(); + if live != 0 && cf_addr != 0 && live != cf_addr { + unsafe { + (*(cf_addr as *mut pyre_interpreter::PyFrame)) + .restore_resume_state_from(&*(live as *const pyre_interpreter::PyFrame)); } - // The committed flush owns the iteration count (the resume pc - // is PAST the FOR_ITER consume); drop any in-flight item so - // the legacy deliver cannot re-apply one. - crate::jitcode_dispatch::fbw_foriter_inflight_clear(); - WALK_END_RESTART_PC.with(|c| c.set(Some(resume_py_pc))); - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); } - let call_forward_abort = match &walk_result { - Err(crate::jitcode_dispatch::DispatchError::AbortPermanentMarkerReached { pc }) => { - Some((*pc, true)) - } - Err( - crate::jitcode_dispatch::DispatchError::LoopBearingCalleeInlineUnsupported { - pc, - }, - ) => Some((*pc, false)), - _ => None, - }; - let mut committed_entry_carrier_call_py_pc = None; - if let Some((abort_jit_pc, is_marker_abort)) = call_forward_abort { - // gh#467: a supported abort fired inside a TOP-level inline - // sub-walk whose callee executed no concrete effect - // (`try_walker_inline_user_call` latched the carrier only under - // that gate). The nested-unjournaled-decline class means the - // residual did not execute; its callee attempt can be discarded - // with any inside-only unjournaled mark. Flush the OUTER frame - // at the CALL that entered the callee and resume the interpreter - // forward — re-executing the whole call from scratch — instead - // of the legacy replay from loop entry, which double-applies the - // non-journaled pre-CALL store. The abort's `abort_jit_pc` is a - // CALLEE coordinate with no meaning in the outer py_pc tables, - // so the outer CALL py_pc and operand stack come from the latch. - // Convergence of `run_blackhole_interp_to_cancel_tracing` - // (`pyjitpl.py:2949`), minus the inner-frame rebuild (#126/#215). - let carrier = crate::jitcode_dispatch::fbw_abort_carrier_clone(); - match carrier.as_ref() { - Some(crate::jitcode_dispatch::InlineAbortCarrier::Entry { - outer_jitcode_index, - call_jitcode_pc, - call_stack, - }) => { - if let Some(call_py_pc) = - resolve_entry_carrier_call_py_pc(*outer_jitcode_index, *call_jitcode_pc) - { - if crate::state::flush_walk_end_state_at_outer_call( - ctx, cf_addr, call_py_pc, call_stack, - ) { - committed_entry_carrier_call_py_pc = Some(call_py_pc); - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] gh#467 CALL-forward COMMIT \ + // The committed flush owns the iteration count (the resume pc + // is PAST the FOR_ITER consume); drop any in-flight item so + // the legacy deliver cannot re-apply one. + crate::jitcode_dispatch::fbw_foriter_inflight_clear(); + WALK_END_RESTART_PC.with(|c| c.set(Some(resume_py_pc))); + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); + } + let call_forward_abort = match &walk_result { + Err(crate::jitcode_dispatch::DispatchError::AbortPermanentMarkerReached { pc }) => { + Some((*pc, true)) + } + Err(crate::jitcode_dispatch::DispatchError::LoopBearingCalleeInlineUnsupported { + pc, + }) => Some((*pc, false)), + _ => None, + }; + let mut committed_entry_carrier_call_py_pc = None; + if let Some((abort_jit_pc, is_marker_abort)) = call_forward_abort { + // gh#467: a supported abort fired inside a TOP-level inline + // sub-walk whose callee executed no concrete effect + // (`try_walker_inline_user_call` latched the carrier only under + // that gate). The nested-unjournaled-decline class means the + // residual did not execute; its callee attempt can be discarded + // with any inside-only unjournaled mark. Flush the OUTER frame + // at the CALL that entered the callee and resume the interpreter + // forward — re-executing the whole call from scratch — instead + // of the legacy replay from loop entry, which double-applies the + // non-journaled pre-CALL store. The abort's `abort_jit_pc` is a + // CALLEE coordinate with no meaning in the outer py_pc tables, + // so the outer CALL py_pc and operand stack come from the latch. + // Convergence of `run_blackhole_interp_to_cancel_tracing` + // (`pyjitpl.py:2949`), minus the inner-frame rebuild (#126/#215). + let carrier = crate::jitcode_dispatch::fbw_abort_carrier_clone(); + match carrier.as_ref() { + Some(crate::jitcode_dispatch::InlineAbortCarrier::Entry { + outer_jitcode_index, + call_jitcode_pc, + call_stack, + }) => { + if let Some(call_py_pc) = + resolve_entry_carrier_call_py_pc(*outer_jitcode_index, *call_jitcode_pc) + { + if crate::state::flush_walk_end_state_at_outer_call( + ctx, cf_addr, call_py_pc, call_stack, + ) { + committed_entry_carrier_call_py_pc = Some(call_py_pc); + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] gh#467 CALL-forward COMMIT \ abort_jit_pc={abort_jit_pc} call_py_pc={call_py_pc} \ stack_depth={}", - call_stack.len() - ); - } - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] gh#467 CALL-forward declined at \ - call_py_pc={call_py_pc} (depth mismatch / unresolved local / \ - lastblock) — legacy replay kept" + call_stack.len() ); } + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( "[fbw-abort-flush] gh#467 CALL-forward declined at \ - abort_jit_pc={abort_jit_pc} (unresolved outer jitcode_index={} \ - or null code ptr) — legacy replay kept", - outer_jitcode_index, + call_py_pc={call_py_pc} (depth mismatch / unresolved local / \ + lastblock) — legacy replay kept" ); } + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] gh#467 CALL-forward declined at \ + abort_jit_pc={abort_jit_pc} (unresolved outer jitcode_index={} \ + or null code ptr) — legacy replay kept", + outer_jitcode_index, + ); } - Some(crate::jitcode_dispatch::InlineAbortCarrier::MidBody(payload)) - if (is_marker_abort + } + Some(crate::jitcode_dispatch::InlineAbortCarrier::MidBody(payload)) + if (is_marker_abort + && payload.abort_kind + == crate::jitcode_dispatch::MidBodyAbortKind::Marker) + || (!is_marker_abort && payload.abort_kind - == crate::jitcode_dispatch::MidBodyAbortKind::Marker) - || (!is_marker_abort - && payload.abort_kind - == crate::jitcode_dispatch::MidBodyAbortKind::Structural) => - { - if let Some(words) = resolve_midbody_flush_words(payload) { - if try_commit_midbody_abort(ctx, cf_addr, payload, words) { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] gh#467 callee-rebuild COMMIT \ + == crate::jitcode_dispatch::MidBodyAbortKind::Structural) => + { + if let Some(words) = resolve_midbody_flush_words(payload) { + if try_commit_midbody_abort(ctx, cf_addr, payload, words) { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] gh#467 callee-rebuild COMMIT \ abort_jit_pc={abort_jit_pc} callee_py_pc={} \ call_py_pc={} post_call_py_pc={}", - words.callee_py_pc, words.call_py_pc, words.post_call_py_pc, - ); - } - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] gh#467 callee-rebuild declined at \ - callee_py_pc={} — legacy replay kept", - words.callee_py_pc, + words.callee_py_pc, words.call_py_pc, words.post_call_py_pc, ); } + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( "[fbw-abort-flush] gh#467 callee-rebuild declined at \ + callee_py_pc={} — legacy replay kept", + words.callee_py_pc, + ); + } + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] gh#467 callee-rebuild declined at \ abort_jit_pc={abort_jit_pc} (unresolved carried jitcode identity \ or null code ptr) — legacy replay kept", + ); + } + } + None if is_marker_abort => { + if crate::jitcode_dispatch::fbw_has_unjournaled_effect() + || session.borrow().abort_in_subwalk + { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ + (unjournaled effect or inline sub-walk) — legacy replay kept" ); } - } - None if is_marker_abort => { - if crate::jitcode_dispatch::fbw_has_unjournaled_effect() - || session.borrow().abort_in_subwalk - { + } else if let Some(resume_py_pc) = + crate::jitcode_dispatch::fbw_abort_resume_py_pc(sym, abort_jit_pc) + { + if crate::state::flush_walk_end_state_to_frame(ctx, cf_addr, resume_py_pc) { if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( - "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ - (unjournaled effect or inline sub-walk) — legacy replay kept" - ); - } - } else if let Some(resume_py_pc) = - crate::jitcode_dispatch::fbw_abort_resume_py_pc(sym, abort_jit_pc) - { - if crate::state::flush_walk_end_state_to_frame( - ctx, - cf_addr, - resume_py_pc, - ) { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] COMMIT abort_jit_pc={abort_jit_pc} \ + "[fbw-abort-flush] COMMIT abort_jit_pc={abort_jit_pc} \ resume_py_pc={resume_py_pc}" - ); - } - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] declined at resume_py_pc={resume_py_pc} \ - (shadow slot without concrete / depth / lastblock) — legacy replay kept" ); } + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] declined at resume_py_pc={resume_py_pc} \ + (shadow slot without concrete / depth / lastblock) — legacy replay kept" + ); } } - _ if crate::jitcode_dispatch::fbw_debug_abort_enabled() => { - eprintln!( - "[fbw-abort-flush] gh#467 CALL-forward declined at \ - abort_jit_pc={abort_jit_pc} (no carrier) — legacy replay kept" - ); - } - _ => {} } - if carrier.is_some() { - crate::jitcode_dispatch::fbw_abort_carrier_clear(); + _ if crate::jitcode_dispatch::fbw_debug_abort_enabled() => { + eprintln!( + "[fbw-abort-flush] gh#467 CALL-forward declined at \ + abort_jit_pc={abort_jit_pc} (no carrier) — legacy replay kept" + ); } + _ => {} } - if let Err( - crate::jitcode_dispatch::DispatchError::LoopBearingCalleeInlineUnsupported { pc }, - ) = &walk_result - { - let abort_jit_pc = *pc; - if !crate::jitcode_dispatch::fbw_executed_nonpure_residual() { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ + if carrier.is_some() { + crate::jitcode_dispatch::fbw_abort_carrier_clear(); + } + } + if let Err(crate::jitcode_dispatch::DispatchError::LoopBearingCalleeInlineUnsupported { + pc, + }) = &walk_result + { + let abort_jit_pc = *pc; + if !crate::jitcode_dispatch::fbw_executed_nonpure_residual() { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ (no executed non-pure residual) — legacy replay kept" - ); - } - } else if crate::jitcode_dispatch::fbw_has_unjournaled_effect() { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ + ); + } + } else if crate::jitcode_dispatch::fbw_has_unjournaled_effect() { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ (unjournaled effect) — legacy replay kept" - ); - } - } else if let Some((jitcode_index, call_jitcode_pc)) = - crate::jitcode_dispatch::fbw_abort_outer_resume_take() - { - let pjc = crate::state::pyjitcode_for_jitcode_index(jitcode_index as i32); - if let Some(pjc) = pjc { - let resume_py_pc = crate::jitcode_dispatch::python_pc_for_jitcode_pc( - &pjc.metadata, - call_jitcode_pc, - ) as usize; - if committed_entry_carrier_call_py_pc == Some(resume_py_pc) { - crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_clear(); + ); + } + } else if let Some((jitcode_index, call_jitcode_pc)) = + crate::jitcode_dispatch::fbw_abort_outer_resume_take() + { + let pjc = crate::state::pyjitcode_for_jitcode_index(jitcode_index as i32); + if let Some(pjc) = pjc { + let resume_py_pc = crate::jitcode_dispatch::python_pc_for_jitcode_pc( + &pjc.metadata, + call_jitcode_pc, + ) as usize; + if committed_entry_carrier_call_py_pc == Some(resume_py_pc) { + crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_clear(); + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] skipped at resume_py_pc={resume_py_pc} \ + (entry carrier already handled same resume)" + ); + } + } else { + // Flush while the overrides stay rooted in + // FBW_ABORT_OUTER_STACK_OVERRIDES (the flush boxes Int/Float + // locals — an allocation that can move the nursery-resident + // override refs; the area walker forwards them in place), + // then clear the cell. + let committed = + crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_with( + |stack_overrides| { + crate::state::flush_walk_end_state_to_frame_with_stack_overrides( + ctx, + cf_addr, + resume_py_pc, + stack_overrides, + ) + }, + ); + crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_clear(); + if committed { if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( - "[fbw-abort-flush] skipped at resume_py_pc={resume_py_pc} \ - (entry carrier already handled same resume)" - ); - } - } else { - // Flush while the overrides stay rooted in - // FBW_ABORT_OUTER_STACK_OVERRIDES (the flush boxes Int/Float - // locals — an allocation that can move the nursery-resident - // override refs; the area walker forwards them in place), - // then clear the cell. - let committed = - crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_with( - |stack_overrides| { - crate::state::flush_walk_end_state_to_frame_with_stack_overrides( - ctx, - cf_addr, - resume_py_pc, - stack_overrides, - ) - }, - ); - crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_clear(); - if committed { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] COMMIT abort_jit_pc={abort_jit_pc} \ + "[fbw-abort-flush] COMMIT abort_jit_pc={abort_jit_pc} \ resume_py_pc={resume_py_pc} (nested inline decline)" - ); - } - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] declined at resume_py_pc={resume_py_pc} \ - (shadow slot without concrete / depth / lastblock) — legacy replay kept" ); } - } - } else { - crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_clear(); - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( - "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ - (unresolved outer jitcode_index={jitcode_index}) — legacy replay kept" + "[fbw-abort-flush] declined at resume_py_pc={resume_py_pc} \ + (shadow slot without concrete / depth / lastblock) — legacy replay kept" ); } } - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ - (no outer caller resume pc) — legacy replay kept" - ); + } else { + crate::jitcode_dispatch::fbw_abort_outer_stack_overrides_clear(); + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ + (unresolved outer jitcode_index={jitcode_index}) — legacy replay kept" + ); + } } + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \ + (no outer caller resume pc) — legacy replay kept" + ); } } @@ -2727,76 +2693,49 @@ fn run_perfn_walk( // no-unjournaled-effect / no-sub-walk predicate and same all-or-nothing // `flush_walk_end_state_to_frame` gate as the CloseLoop / marker legs; // when the flush declines (a slot the shadow cannot resolve) the legacy - // drop stands (the residual S3 case). `PYRE_FBW_BRANCH_FLUSH=0` opts - // out. - if std::env::var_os("PYRE_FBW_BRANCH_FLUSH").as_deref() != Some(std::ffi::OsStr::new("0")) { - let kept_stack_abort_pc = match &walk_result { - Err( - crate::jitcode_dispatch::DispatchError::BranchGuardUnrestorableKeptStackPermanent { - pc, - }, - ) => Some((*pc, false)), - Err(crate::jitcode_dispatch::DispatchError::BranchGuardKeptStackUnsupported { + // drop stands (the residual S3 case). + let kept_stack_abort_pc = match &walk_result { + Err( + crate::jitcode_dispatch::DispatchError::BranchGuardUnrestorableKeptStackPermanent { pc, - }) => Some((*pc, true)), - _ => None, - }; - if let Some((pc, is_unsupported)) = kept_stack_abort_pc { - let abort_jit_pc = pc; - if crate::jitcode_dispatch::fbw_has_unjournaled_effect() - || session.borrow().abort_in_subwalk - { - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-branch-flush] declined at abort_jit_pc={abort_jit_pc} \ + }, + ) => Some((*pc, false)), + Err(crate::jitcode_dispatch::DispatchError::BranchGuardKeptStackUnsupported { pc }) => { + Some((*pc, true)) + } + _ => None, + }; + if let Some((pc, is_unsupported)) = kept_stack_abort_pc { + let abort_jit_pc = pc; + if crate::jitcode_dispatch::fbw_has_unjournaled_effect() + || session.borrow().abort_in_subwalk + { + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-branch-flush] declined at abort_jit_pc={abort_jit_pc} \ (unjournaled effect or inline sub-walk) — legacy drop kept" - ); - } - } else if let Some(resume_py_pc) = - crate::jitcode_dispatch::fbw_abort_resume_py_pc(sym, abort_jit_pc) - { - // Two kept-stack branch aborts reach this leg (`is_unsupported` - // came from the `kept_stack_abort_pc` match). Both resume at a - // FOR_ITER header whose walk already advanced the iterator; they - // differ in whether the consumed item's body ran. - let committed = if is_unsupported { - if crate::jitcode_dispatch::fbw_foriter_inflight_completed_at_resume( - cf_addr, - resume_py_pc, - ) { - // The consumed item's body already ran, so resume at - // the FOR_ITER header without re-delivering it. - crate::state::flush_walk_end_state_to_frame(ctx, cf_addr, resume_py_pc) - } else { - // A nested inner FOR_ITER can carry the enclosing - // iterator as its kept stack before the consumed - // item's body runs. Mirror Shape A and deliver that - // in-flight item exactly once. - let push = - crate::jitcode_dispatch::fbw_foriter_inflight_take_for_resume( - cf_addr, - resume_py_pc, - ); - push.is_some() - && crate::state::flush_walk_end_state_to_frame_with_item( - ctx, - cf_addr, - resume_py_pc, - push, - ) - } + ); + } + } else if let Some(resume_py_pc) = + crate::jitcode_dispatch::fbw_abort_resume_py_pc(sym, abort_jit_pc) + { + // Two kept-stack branch aborts reach this leg (`is_unsupported` + // came from the `kept_stack_abort_pc` match). Both resume at a + // FOR_ITER header whose walk already advanced the iterator; they + // differ in whether the consumed item's body ran. + let committed = if is_unsupported { + if crate::jitcode_dispatch::fbw_foriter_inflight_completed_at_resume( + cf_addr, + resume_py_pc, + ) { + // The consumed item's body already ran, so resume at + // the FOR_ITER header without re-delivering it. + crate::state::flush_walk_end_state_to_frame(ctx, cf_addr, resume_py_pc) } else { - // Shape A — a `BranchGuardUnrestorableKeptStackPermanent` - // abort resumes AT a FOR_ITER header whose consumed item is - // in flight (`body_pc == resume_py_pc + 1`, the opcode - // there really is a FOR_ITER): the walk advanced the - // iterator but the item is not yet on the flushed (header) - // stack, so deliver it (push + reposition to the body) so - // the body runs once. Commit ONLY when an item is - // delivered — a Permanent abort not at such a header keeps - // the legacy drop byte-identically (the residual S3 case), - // so every other abort shape (and the whole flag-OFF path) - // is untouched. + // A nested inner FOR_ITER can carry the enclosing + // iterator as its kept stack before the consumed + // item's body runs. Mirror Shape A and deliver that + // in-flight item exactly once. let push = crate::jitcode_dispatch::fbw_foriter_inflight_take_for_resume( cf_addr, resume_py_pc, @@ -2808,25 +2747,48 @@ fn run_perfn_walk( resume_py_pc, push, ) - }; - if committed { - // The flush owns the iteration count; drop any remaining - // in-flight items so the legacy deliver cannot re-apply - // one (exactly-once). - crate::jitcode_dispatch::fbw_foriter_inflight_clear(); - WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); - if crate::jitcode_dispatch::fbw_debug_abort_enabled() { - eprintln!( - "[fbw-branch-flush] COMMIT abort_jit_pc={abort_jit_pc} \ - resume_py_pc={resume_py_pc} (delivered in-flight FOR_ITER item)" - ); - } - } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + } + } else { + // Shape A — a `BranchGuardUnrestorableKeptStackPermanent` + // abort resumes AT a FOR_ITER header whose consumed item is + // in flight (`body_pc == resume_py_pc + 1`, the opcode + // there really is a FOR_ITER): the walk advanced the + // iterator but the item is not yet on the flushed (header) + // stack, so deliver it (push + reposition to the body) so + // the body runs once. Commit ONLY when an item is + // delivered — a Permanent abort not at such a header keeps + // the legacy drop byte-identically (the residual S3 case), + // so every other abort shape (and the whole flag-OFF path) + // is untouched. + let push = crate::jitcode_dispatch::fbw_foriter_inflight_take_for_resume( + cf_addr, + resume_py_pc, + ); + push.is_some() + && crate::state::flush_walk_end_state_to_frame_with_item( + ctx, + cf_addr, + resume_py_pc, + push, + ) + }; + if committed { + // The flush owns the iteration count; drop any remaining + // in-flight items so the legacy deliver cannot re-apply + // one (exactly-once). + crate::jitcode_dispatch::fbw_foriter_inflight_clear(); + WALK_END_FLUSH_COMMITTED.with(|c| c.set(true)); + if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( - "[fbw-branch-flush] declined at resume_py_pc={resume_py_pc} \ - (shadow slot without concrete / depth / lastblock) — legacy drop kept" + "[fbw-branch-flush] COMMIT abort_jit_pc={abort_jit_pc} \ + resume_py_pc={resume_py_pc} (delivered in-flight FOR_ITER item)" ); } + } else if crate::jitcode_dispatch::fbw_debug_abort_enabled() { + eprintln!( + "[fbw-branch-flush] declined at resume_py_pc={resume_py_pc} \ + (shadow slot without concrete / depth / lastblock) — legacy drop kept" + ); } } } @@ -2858,8 +2820,8 @@ fn run_perfn_walk( // predicate, the journal commit below, and the caller's // consume-vs-rewind) stay in agreement. A multiframe resume is never // armed, so it stays on the legacy rewind-and-replay path. - let terminate_no_replay = crate::jitcode_dispatch::fbw_no_replay_exit_enabled() - && (!is_bridge_trace || crate::jitcode_dispatch::fbw_bridge_noreplay_armed()) + let terminate_no_replay = (!is_bridge_trace + || crate::jitcode_dispatch::fbw_bridge_noreplay_armed()) && matches!( &walk_result, Ok((crate::jitcode_dispatch::DispatchOutcome::Terminate, _)) @@ -3551,7 +3513,7 @@ fn full_body_walk_trace( // Clear the walk-local bool-box-truth map left by a prior aborted walk so // it cannot leak into this one. crate::jitcode_dispatch::bool_box_truth_reset(); - // Slice b (PYRE_FBW_CALL_ASSEMBLER): clear any Finish payload a prior + // Slice b: clear any Finish payload a prior // aborted walk's top-level `*_return` arm may have stashed, so a stale // value cannot leak into this walk's `Terminate` handling. crate::jitcode_dispatch::fbw_finish_payload_reset(); @@ -3632,15 +3594,13 @@ fn full_body_walk_trace( } crate::jitcode_dispatch::DispatchOutcome::Terminate => { // A loop-free portal exit: the top-level `*_return` reached - // `done_with_this_frame` with no back-edge. Under the - // PYRE_FBW_CALL_ASSEMBLER gate the return arm routed through - // `fbw_terminate_with_finish`, which re-boxed the result to - // Type::Ref, recorded the vable store-back + GUARD_NOT_FORCED_2, - // and stashed the finish payload. Build the portal-exit FINISH - // from it so the compile pipeline records FINISH from - // `finish_args` (matching `StepResult::Return` in - // trace_opcode.rs). Ungated → no payload → `Abort` - // exactly as before the slice. + // `done_with_this_frame` with no back-edge. The return arm + // routed through `fbw_terminate_with_finish`, which re-boxed the + // result to Type::Ref, recorded the vable store-back + + // GUARD_NOT_FORCED_2, and stashed the finish payload. Build the + // portal-exit FINISH from it so the compile pipeline records + // FINISH from `finish_args` (matching `StepResult::Return` in + // trace_opcode.rs). No payload → `Abort`. let finish_is_exception = crate::jitcode_dispatch::fbw_finish_is_exception(); match crate::jitcode_dispatch::fbw_finish_payload_take() { // A top-level `void_return/` stashes a `Type::Void`-marked @@ -3728,7 +3688,7 @@ fn full_body_walk_trace( | DE::LoopBearingCalleeInlineUnsupported { .. } | DE::UnfoldableListAppendResidualUnsupported { .. } | DE::ResidualCallArgUnbound { .. } => TraceAction::Abort, - // #68 multiframe (`PYRE_FBW_INLINE_MULTIFRAME`): a data-dependent + // #68 multiframe: a data-dependent // `goto_if_not` whose branch input is not concrete at trace-time // recurs identically on every retrace of this entry (the same // jitcode walked from the same start_pc reaches the same @@ -3738,12 +3698,8 @@ fn full_body_walk_trace( // such a branch, which would otherwise re-trace unbounded (each // re-walk executes the body's residual calls before failing) — // an unbounded slowdown. Decline it so the location interprets - // instead. Gated on the flag so the default path's plain - // `Abort` (a capability landing mid-run can still pick it up) is - // byte-identical. - DE::GotoIfNotValueNotConcrete { .. } - if crate::jitcode_dispatch::fbw_inline_multiframe_enabled() => - { + // instead. + DE::GotoIfNotValueNotConcrete { .. } => { fbw_decline(crate::driver::make_green_key(w_code, start_pc)); TraceAction::Abort } diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index 89b31ae1395..8f08484c8c2 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -2889,7 +2889,7 @@ pub fn trace_and_compile_from_bridge( driver.last_bridge_is_exception_guard }; if last_bridge_is_exception_guard { - // With `PYRE_EXC_EDGE_BRIDGE` the walker emits the whole exception + // The walker emits the whole exception // resumption sequence (SAVE_EXC_CLASS/SAVE_EXCEPTION/RESTORE_EXCEPTION + // a snapshotted GUARD_EXCEPTION) at the bridge-entry frame state, where // the guard can capture resume data. The legacy call-site prologue @@ -3013,7 +3013,7 @@ pub fn trace_and_compile_from_bridge( pyre_interpreter::pycode::lookup_exceptiontable(&code.exceptiontable, off).is_some() } }; - // Exception-edge bridge (`PYRE_EXC_EDGE_BRIDGE`): route the caught-in-frame + // Exception-edge bridge: route the caught-in-frame // single-frame resume to the in-frame `except` handler (walker // `find_catch_before_resume_live`) instead of declining. The escaping case // (uncaught) and the multi-frame resume still decline here — those are diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index b67aa806aa7..19090f78a66 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -54,15 +54,6 @@ fn local_to_vable_slot(var_num: usize) -> usize { var_num } -/// `PYRE_FBW_DELETE_FAST` gates the walker-native DELETE_FAST lowering. -/// Default ON; `0`/`false` restores the permanent-abort marker. -#[inline] -fn fbw_delete_fast_enabled() -> bool { - std::env::var("PYRE_FBW_DELETE_FAST") - .map(|v| v != "0" && !v.eq_ignore_ascii_case("false")) - .unwrap_or(true) -} - /// Re-export of `pyre_jit_trace::pyjitcode::portal_red_pre_regalloc_slots` /// so the codewriter pipeline shares the same formula with the /// portal-bridge install path in `canonical_bridge.rs`. See the @@ -11984,110 +11975,26 @@ impl CodeWriter { } Instruction::DeleteFast { var_num } => { - if fbw_delete_fast_enabled() { - let idx = var_num.get(op_arg).as_usize(); - let local_slot = local_to_vable_slot(idx) as i64; - let v_idx: super::flow::FlowValue = - super::flow::Constant::signed(local_slot).into(); - let code_const: super::flow::FlowValue = - super::flow::Constant::new( - super::flow::ConstantValue::Signed(w_code as i64), - Some(Kind::Ref), - ) - .into(); - let name_idx_const: super::flow::FlowValue = - super::flow::Constant::signed(idx as i64).into(); - // pyopcode.py:998 DELETE_FAST checks for an - // unbound local before clearing its slot. A - // statically unbound slot therefore raises - // unconditionally and performs no write. - if matches!( - current_state.local_value_at(idx), - Some(super::flow::FlowValue::Constant(c)) - if c.value == super::flow::ConstantValue::None - ) { - let v_li: super::flow::FlowValue = - super::flow::Constant::signed(py_pc as i64 - 1).into(); - record_graph_op( - ¤t_block.block(), - "setfield_vable_i", - vable_setfield_int_graph_args( - frame_var.into(), - v_li.into(), - VABLE_LAST_INSTR_FIELD_IDX, - ), - None, - py_pc as i64, - ); - let exc_value = emit_graph_op_with_result( - &mut graph, - ¤t_block.block(), - "unbound_local_error", - vec![code_const.into(), name_idx_const.into()], - Kind::Ref, - py_pc as i64, - ); - let exc_flow: super::flow::FlowValue = exc_value.into(); - emit_raise!(0u16, exc_flow, py_pc as i64, true); - continue; - } - // The dynamic bound check below splits the - // exception arm explicitly. Its successful - // continuation cannot raise and must not - // receive the generic per-op catch edge. - exception_edge_handled = true; - let idx_u16 = idx as u16; - emit_load_fast_ref!(current_depth, idx_u16, py_pc); - let _value_reg = emit_popvalue_ref!(current_depth, py_pc); - let value_value = pop_ref_or_fresh(&mut current_state, &mut graph); - let is_null = emit_graph_op_with_result( - &mut graph, - ¤t_block.block(), - "ptr_iszero", - vec![value_value.into()], - Kind::Int, - py_pc as i64, - ); - current_block.block().borrow_mut().exitswitch = - Some(super::flow::ExitSwitch::Value(is_null.into())); - let unbound_state = current_state.clone(); - let unbound_block = SpamBlockRef::new( - graph.new_block(Vec::new()), - Some(unbound_state.clone()), - ); - unbound_block.block().borrow_mut().inputargs = - unbound_state.getvariables(); - all_walker_blocks.push(unbound_block.clone()); - append_exit( - ¤t_block.block(), - output_link( - ¤t_state, - &unbound_state, - unbound_block.block(), - ), - ); - set_last_bool_exitcase(¤t_block.block(), true); - let bound_state = current_state.clone(); - let bound_block = SpamBlockRef::new( - graph.new_block(Vec::new()), - Some(bound_state.clone()), - ); - bound_block.block().borrow_mut().inputargs = - bound_state.getvariables(); - all_walker_blocks.push(bound_block.clone()); - append_exit( - ¤t_block.block(), - output_link(¤t_state, &bound_state, bound_block.block()), - ); - set_last_bool_exitcase(¤t_block.block(), false); - - // The unbound arm raises before any clear, - // matching pyopcode.py:998 DELETE_FAST. Keep - // last_instr at the deleting instruction so - // exception unwind and deopt share its frame - // coordinate. - current_block = unbound_block; - current_state = unbound_state; + let idx = var_num.get(op_arg).as_usize(); + let local_slot = local_to_vable_slot(idx) as i64; + let v_idx: super::flow::FlowValue = + super::flow::Constant::signed(local_slot).into(); + let code_const: super::flow::FlowValue = super::flow::Constant::new( + super::flow::ConstantValue::Signed(w_code as i64), + Some(Kind::Ref), + ) + .into(); + let name_idx_const: super::flow::FlowValue = + super::flow::Constant::signed(idx as i64).into(); + // pyopcode.py:998 DELETE_FAST checks for an + // unbound local before clearing its slot. A + // statically unbound slot therefore raises + // unconditionally and performs no write. + if matches!( + current_state.local_value_at(idx), + Some(super::flow::FlowValue::Constant(c)) + if c.value == super::flow::ConstantValue::None + ) { let v_li: super::flow::FlowValue = super::flow::Constant::signed(py_pc as i64 - 1).into(); record_graph_op( @@ -12110,79 +12017,151 @@ impl CodeWriter { py_pc as i64, ); let exc_flow: super::flow::FlowValue = exc_value.into(); - emit_raise!(0u16, exc_flow.clone(), py_pc as i64, true); - if let Some(catch_label) = - catch_for_pc.get(py_pc).copied().flatten() - { - let site = catch_sites - .iter() - .find(|site| site.landing_label == catch_label) - .expect("catch site for DELETE_FAST raise"); - let link = current_block - .block() - .borrow() - .exits - .last() - .cloned() - .expect("DELETE_FAST raise catch edge"); - carry_explicit_raise_value_on_catch_stack( - &link, - &site.landing, - exc_flow, - ); - } + emit_raise!(0u16, exc_flow, py_pc as i64, true); + continue; + } + // The dynamic bound check below splits the + // exception arm explicitly. Its successful + // continuation cannot raise and must not + // receive the generic per-op catch edge. + exception_edge_handled = true; + let idx_u16 = idx as u16; + emit_load_fast_ref!(current_depth, idx_u16, py_pc); + let _value_reg = emit_popvalue_ref!(current_depth, py_pc); + let value_value = pop_ref_or_fresh(&mut current_state, &mut graph); + let is_null = emit_graph_op_with_result( + &mut graph, + ¤t_block.block(), + "ptr_iszero", + vec![value_value.into()], + Kind::Int, + py_pc as i64, + ); + current_block.block().borrow_mut().exitswitch = + Some(super::flow::ExitSwitch::Value(is_null.into())); + let unbound_state = current_state.clone(); + let unbound_block = SpamBlockRef::new( + graph.new_block(Vec::new()), + Some(unbound_state.clone()), + ); + unbound_block.block().borrow_mut().inputargs = + unbound_state.getvariables(); + all_walker_blocks.push(unbound_block.clone()); + append_exit( + ¤t_block.block(), + output_link(¤t_state, &unbound_state, unbound_block.block()), + ); + set_last_bool_exitcase(¤t_block.block(), true); + let bound_state = current_state.clone(); + let bound_block = SpamBlockRef::new( + graph.new_block(Vec::new()), + Some(bound_state.clone()), + ); + bound_block.block().borrow_mut().inputargs = bound_state.getvariables(); + all_walker_blocks.push(bound_block.clone()); + append_exit( + ¤t_block.block(), + output_link(¤t_state, &bound_state, bound_block.block()), + ); + set_last_bool_exitcase(¤t_block.block(), false); - // The bound arm is the continuing block. The - // clear is one PY_NULL write, matching - // pyopcode.py:998 DELETE_FAST's single slot - // assignment after the check succeeds. - current_block = bound_block; - current_state = bound_state; - needs_fallthrough = true; - record_graph_op( - ¤t_block.block(), - "setarrayitem_vable_r", - vable_setarrayitem_ref_graph_args( - frame_var.into(), - v_idx.into(), - super::flow::Constant::none().into(), - ), - None, - py_pc as i64, - ); - // `framestate.py:105-114 union`: an undefined - // local is `None`, not a Constant(None). The - // graph-side vable write above still carries - // the runtime PY_NULL sentinel. - current_state.clear_local_value(idx); - // The successful branch egg has now executed - // the clear. Close it into the next bytecode - // through the ordinary joinpoint machinery, - // exactly as `flowcontext.py:424-475 - // mergeblock` closes every branch arrival. - // Directly inserting this block into the - // candidate list strands an older candidate - // at the same PC when conditional control flow - // already reaches the continuation. - let next_py_pc = py_pc + 1; - let mut continuation_state = current_state.clone(); - continuation_state.next_offset = next_py_pc; - continuation_state.blocklist = - frame_blocks_for_offset(code, next_py_pc); - let _ = mergeblock( - code, - &mut graph, - &mut joinpoints, - ¤t_block, - &continuation_state, - next_py_pc, - &mut pendingblocks, - &mut all_walker_blocks, + // The unbound arm raises before any clear, + // matching pyopcode.py:998 DELETE_FAST. Keep + // last_instr at the deleting instruction so + // exception unwind and deopt share its frame + // coordinate. + current_block = unbound_block; + current_state = unbound_state; + let v_li: super::flow::FlowValue = + super::flow::Constant::signed(py_pc as i64 - 1).into(); + record_graph_op( + ¤t_block.block(), + "setfield_vable_i", + vable_setfield_int_graph_args( + frame_var.into(), + v_li.into(), + VABLE_LAST_INSTR_FIELD_IDX, + ), + None, + py_pc as i64, + ); + let exc_value = emit_graph_op_with_result( + &mut graph, + ¤t_block.block(), + "unbound_local_error", + vec![code_const.into(), name_idx_const.into()], + Kind::Ref, + py_pc as i64, + ); + let exc_flow: super::flow::FlowValue = exc_value.into(); + emit_raise!(0u16, exc_flow.clone(), py_pc as i64, true); + if let Some(catch_label) = catch_for_pc.get(py_pc).copied().flatten() { + let site = catch_sites + .iter() + .find(|site| site.landing_label == catch_label) + .expect("catch site for DELETE_FAST raise"); + let link = current_block + .block() + .borrow() + .exits + .last() + .cloned() + .expect("DELETE_FAST raise catch edge"); + carry_explicit_raise_value_on_catch_stack( + &link, + &site.landing, + exc_flow, ); - needs_fallthrough = false; - } else { - emit_abort_permanent!(py_pc); } + + // The bound arm is the continuing block. The + // clear is one PY_NULL write, matching + // pyopcode.py:998 DELETE_FAST's single slot + // assignment after the check succeeds. + current_block = bound_block; + current_state = bound_state; + needs_fallthrough = true; + record_graph_op( + ¤t_block.block(), + "setarrayitem_vable_r", + vable_setarrayitem_ref_graph_args( + frame_var.into(), + v_idx.into(), + super::flow::Constant::none().into(), + ), + None, + py_pc as i64, + ); + // `framestate.py:105-114 union`: an undefined + // local is `None`, not a Constant(None). The + // graph-side vable write above still carries + // the runtime PY_NULL sentinel. + current_state.clear_local_value(idx); + // The successful branch egg has now executed + // the clear. Close it into the next bytecode + // through the ordinary joinpoint machinery, + // exactly as `flowcontext.py:424-475 + // mergeblock` closes every branch arrival. + // Directly inserting this block into the + // candidate list strands an older candidate + // at the same PC when conditional control flow + // already reaches the continuation. + let next_py_pc = py_pc + 1; + let mut continuation_state = current_state.clone(); + continuation_state.next_offset = next_py_pc; + continuation_state.blocklist = + frame_blocks_for_offset(code, next_py_pc); + let _ = mergeblock( + code, + &mut graph, + &mut joinpoints, + ¤t_block, + &continuation_state, + next_py_pc, + &mut pendingblocks, + &mut all_walker_blocks, + ); + needs_fallthrough = false; } // DELETE_DEREF clears the cell contents after checking From e6899fa02fe2249cc753d2e090bf8973c4c9e3bc Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 24 Jul 2026 04:32:30 +0900 Subject: [PATCH 08/11] gc: re-arm the frame-array write barrier after every flush store The walk-end / abort flush functions box each Int/Float slot (an allocation that can trigger a minor collection) one at a time into the detached frame array. The array is forwarded by a minor collection only while it is in the remembered set, and each minor consumes that entry, so a single barrier after the whole loop left a window: a nursery Ref stored in one iteration could be dropped by a minor collection triggered by the next iteration's boxing before the array was re-armed, leaving a stale pointer in the resumed frame. Re-arm the barrier after every store (and before the first allocation that follows a pre-loop nursery store) in flush_walk_end_state_to_frame_inner, flush_walk_end_state_at_outer_call, write_back_outer_locals, and flush_walk_end_state_after_outer_call. Assisted-by: Claude --- pyre/pyre-jit-trace/src/state.rs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index fc6539567dd..1ceba97d892 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -4273,10 +4273,10 @@ fn flush_walk_end_state_to_frame_inner( } // Commit one slot at a time, re-reading the shadow entry per slot: // boxing an Int/Float slot allocates and may trigger a minor - // collection, which moves nursery objects — the trace-ctx forwarding - // hook keeps the shadow entries current, and each already-written - // slot is reachable from the (rooted) frame, so neither side goes - // stale across the loop. + // collection. A nursery Ref already written into the detached frame + // array is forwarded only while the array is in the remembered set, + // and each minor consumes that entry, so re-arm the barrier after + // every store — before the next iteration's boxing can allocate. for abs in 0..live { let boxed = if abs >= nlocals && have_overrides { // The override is the authoritative caller CALL stack. Its @@ -4291,6 +4291,7 @@ fn flush_walk_end_state_to_frame_inner( unsafe { (*arr_ptr).as_mut_slice()[abs] = boxed; } + frame_array_write_barrier(frame as *mut u8, arr_ptr); } unsafe { let pf = &mut *(frame as *mut PyFrame); @@ -4424,16 +4425,16 @@ pub(crate) fn flush_walk_end_state_at_outer_call( } // Commit the operand stack FIRST: `call_stack` holds live nursery-resident // refs, and boxing an Int/Float local below can trigger a minor collection. - // Once written into the (rooted) frame array they are forwarded with it, so - // landing them before any allocation keeps them current. + // The detached frame array is forwarded only while it is in the remembered + // set, so arm the barrier once the stack refs are landed and again after + // every local store (each minor consumes the remembered entry). for (i, &value) in call_stack.iter().enumerate() { unsafe { (*arr_ptr).as_mut_slice()[nlocals + i] = value; } } - // Commit the locals from the shadow (re-reading per slot: boxing may move - // nursery objects, but each written slot is frame-reachable and the - // trace-ctx forwarding hook keeps the shadow current). + frame_array_write_barrier(frame as *mut u8, arr_ptr); + // Commit the locals from the shadow, re-reading per slot. for abs in 0..nlocals { let Some((_opref, value)) = ctx.virtualizable_entry_at(base + abs) else { return false; @@ -4442,6 +4443,7 @@ pub(crate) fn flush_walk_end_state_at_outer_call( unsafe { (*arr_ptr).as_mut_slice()[abs] = boxed; } + frame_array_write_barrier(frame as *mut u8, arr_ptr); } unsafe { let pf = &mut *(frame as *mut PyFrame); @@ -4538,6 +4540,9 @@ pub(crate) fn write_back_outer_locals(ctx: &TraceCtx, frame: usize) -> bool { return false; } let base = info.num_static_extra_boxes; + // Boxing an Int/Float slot allocates; the detached frame array is + // forwarded only while it is in the remembered set, and each minor + // consumes that entry, so re-arm the barrier after every store. for abs in 0..nlocals { let Some((_opref, value)) = ctx.virtualizable_entry_at(base + abs) else { return false; @@ -4546,6 +4551,7 @@ pub(crate) fn write_back_outer_locals(ctx: &TraceCtx, frame: usize) -> bool { unsafe { (*arr_ptr).as_mut_slice()[abs] = boxed; } + frame_array_write_barrier(frame as *mut u8, arr_ptr); } frame_array_write_barrier(frame as *mut u8, arr_ptr); true @@ -4578,10 +4584,13 @@ pub(crate) fn flush_walk_end_state_after_outer_call( *(frame_ptr.add(PYFRAME_LOCALS_CELLS_STACK_OFFSET) as *const *mut pyre_object::FixedObjectArray) }; - // Land the nursery-resident result before boxing locals can collect. + // Land the nursery-resident result, then arm the barrier before + // `write_back_outer_locals` boxes locals (an allocation that can + // collect the just-stored result if the array is not remembered). unsafe { (*arr_ptr).as_mut_slice()[nlocals] = retval; } + frame_array_write_barrier(frame as *mut u8, arr_ptr); if !write_back_outer_locals(ctx, frame) { return false; } From 7c9c558d12b87cbd6843d6fbb7204fab1a5e3a35 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 24 Jul 2026 06:36:41 +0900 Subject: [PATCH 09/11] jit(gc): only strip a bridge's save/restore-exception prefix when its result is unused remove_bridge_exception stripped a leading SaveExcClass + SaveException + RestoreException prefix unconditionally (rewrite.py:988), leaving its `XXX should check if the boxes are used later` deferred. A routed exception-guard handler bridge records that same prefix but keeps the SaveException result as last_exc_value for handler code (`except E as e`), so an unconditional strip drops an operand a later op still references. Scan the ops after the prefix for a use of the RestoreException class/value operands (args and failargs) and strip only when neither is reused. Add regression tests for the strip-when-unused and keep-when-reused cases. Assisted-by: Claude --- majit/majit-gc/src/rewrite.rs | 82 +++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/majit/majit-gc/src/rewrite.rs b/majit/majit-gc/src/rewrite.rs index 65843c53a24..5bba51b1bc2 100644 --- a/majit/majit-gc/src/rewrite.rs +++ b/majit/majit-gc/src/rewrite.rs @@ -2869,6 +2869,15 @@ impl GcRewriterImpl { /// rewrite.py:988-1001 remove_bridge_exception: check a common /// case where SaveExcClass + SaveException + RestoreException /// appear at the start of a bridge and are unused. Strip them. + /// + /// rewrite.py:991 leaves an `XXX should check if the boxes are used + /// later; but we just assume they aren't for now`. A routed + /// exception-guard handler bridge breaks that assumption: it records the + /// same prefix but keeps the SaveException result as `last_exc_value` + /// for handler code (`except E as e`), so stripping the prefix would + /// leave a dangling reference. Do the deferred use-check — the + /// RESTORE_EXCEPTION carries the canonical class/value operands, so only + /// strip when no op after the prefix reuses either. fn remove_bridge_exception(ops: &[Op]) -> Vec { let mut start = 0; if ops @@ -2882,10 +2891,24 @@ impl GcRewriterImpl { && ops[start + 1].opcode == OpCode::SaveException && ops[start + 2].opcode == OpCode::RestoreException { - let mut result = Vec::with_capacity(ops.len() - 3); - result.extend_from_slice(&ops[..start]); - result.extend_from_slice(&ops[start + 3..]); - return result; + let restore = &ops[start + 2]; + let class_ref = restore.arg(0); + let value_ref = restore.arg(1); + let used_later = ops[start + 3..].iter().any(|op| { + (0..op.num_args()).any(|k| { + let a = op.arg(k); + a.same_box(&class_ref) || a.same_box(&value_ref) + }) || op.getfailargs().is_some_and(|fa| { + fa.iter() + .any(|a| a.same_box(&class_ref) || a.same_box(&value_ref)) + }) + }); + if !used_later { + let mut result = Vec::with_capacity(ops.len() - 3); + result.extend_from_slice(&ops[..start]); + result.extend_from_slice(&ops[start + 3..]); + return result; + } } ops.to_vec() } @@ -5487,4 +5510,55 @@ mod tests { "JIT_DEBUG must not route its ConstPtr through the gc_table" ); } + + #[test] + fn remove_bridge_exception_strips_unused_prefix() { + use std::rc::Rc; + let class = Rc::new(Op::new(OpCode::SaveExcClass, &[])); + let value = Rc::new(Op::new(OpCode::SaveException, &[])); + let restore = Op::new( + OpCode::RestoreException, + &[ + Operand::from_bound_op(&class), + Operand::from_bound_op(&value), + ], + ); + let ops = vec![ + Op::new(OpCode::SaveExcClass, &[]), + Op::new(OpCode::SaveException, &[]), + restore, + Op::new(OpCode::Finish, &[]), + ]; + let out = GcRewriterImpl::remove_bridge_exception(&ops); + // No op reuses the saved class/value, so the prefix is stripped. + assert_eq!(out.len(), 1); + assert_eq!(out[0].opcode, OpCode::Finish); + } + + #[test] + fn remove_bridge_exception_keeps_prefix_when_value_reused() { + use std::rc::Rc; + let class = Rc::new(Op::new(OpCode::SaveExcClass, &[])); + let value = Rc::new(Op::new(OpCode::SaveException, &[])); + let restore = Op::new( + OpCode::RestoreException, + &[ + Operand::from_bound_op(&class), + Operand::from_bound_op(&value), + ], + ); + // A routed handler bridge keeps the saved value as `last_exc_value` + // for `except E as e`, so a later op still references it. + let consumer = Op::new(OpCode::SameAsR, &[Operand::from_bound_op(&value)]); + let ops = vec![ + Op::new(OpCode::SaveExcClass, &[]), + Op::new(OpCode::SaveException, &[]), + restore, + consumer, + ]; + let out = GcRewriterImpl::remove_bridge_exception(&ops); + // The reused value pins the whole prefix in place. + assert_eq!(out.len(), 4); + assert_eq!(out[0].opcode, OpCode::SaveExcClass); + } } From 912b42c1cae68fc09094f554ad6d8a086a18b034 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 24 Jul 2026 14:58:39 +0900 Subject: [PATCH 10/11] jit: wrap the FBW end-flush loop-close call to satisfy rustfmt The rebase renamed `flush_walk_end_state_to_frame` to `flush_walk_loop_end_state_to_frame` in the end-flush block, pushing the `else if` condition past the 100-column limit; rustfmt moves the opening brace to its own line. Assisted-by: Claude --- pyre/pyre-jit-trace/src/trace.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index a9b85b43cc1..1f53c195ffe 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -2388,7 +2388,8 @@ fn run_perfn_walk( (unjournaled effect) — legacy replay kept" ); } - } else if crate::state::flush_walk_loop_end_state_to_frame(ctx, cf_addr, header_pc) { + } else if crate::state::flush_walk_loop_end_state_to_frame(ctx, cf_addr, header_pc) + { if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( "[fbw-end-flush] COMMIT header_pc={header_pc} bridge={} \ From ebb87c5758f6c9984c80a4dad539eb149ea19e4b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 25 Jul 2026 01:57:49 +0900 Subject: [PATCH 11/11] jit(cranelift): drop the post-staging attached-bridge dispatch for exception guards emit_guard_exit dispatched the attached bridge twice for a can_have_bridge + must_save_exception guard: once before the _store_and_reset_exception staging and once after it. The pre-staging dispatch was widened to cover must_save guards in 618871efb8, so its guard set now supersets the post-staging block's. The post-staging dispatch enters the bridge with the exception globals already cleared, so the bridge entry flavor guard (prepare_resume_from_failure) reads no pending exception; a bridge installed between the two probes would take that wrong-flavor path. dynasm's patched guard jump enters the bridge before the failure-recovery stub stages jf_guard_exc (patch_jump_for_descr, x86/assembler.py:987), matching the pre-staging dispatch. Remove the redundant second dispatch. check.py 304/304 on cranelift. Assisted-by: Claude --- majit/majit-backend-cranelift/src/compiler.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index 090f70e6dce..3b9c24db487 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -6813,22 +6813,6 @@ fn emit_guard_exit( .ins() .store(MemFlags::trusted(), zero, exc_type_addr, 0); } - if info.can_have_bridge && info.must_save_exception { - // Exception guards need the exception payload saved before a bridge - // sees the frame. Do the bridge check before the deadframe-only - // write-barrier/descr stores: dynasm patches exception guards to jump - // to the bridge after staging jf_guard_exc, and the bridge prologue - // immediately re-roots the same frame on the shadow stack. A bridge - // hit therefore does not need the host-call write barrier that only - // protects the returned deadframe path. - emit_attached_bridge_dispatch( - builder, - jf_ptr, - info.bridge_cache_addrs - .expect("can_have_bridge=true GuardInfo must carry bridge_cache_addrs"), - ptr_type, - ); - } if info.gcmap != 0 || info.must_save_exception { // aarch64/assembler.py:967-980 `_reload_frame_if_necessary`: // RPython emits a JITFrame write-barrier after any potentially-