jit: root the blackhole virtualizable_ptr slot, drop the re-entrant trace-too-long teardown, and give the exception-edge bridge its discarded-frame traceback nodes

majit/majit-metainterp/src/blackhole.rs

@@ -1482,6 +1482,26 @@ impl BlackholeInterpreter {
         unsafe {
             let mut current = Some(&mut *self);
             while let Some(frame) = current {
+                // `virtualizable_ptr` is a bare copy of the frame red this
+                // level was bound to, and the banks rooted just above do not
+                // cover it: a collection forwards the register the copy came
+                // from, never the copy.  A young virtualizable makes that
+                // difference observable — a compiled trace allocates an
+                // inlined callee's `PyFrame` with its own `NewWithVtable`,
+                // which the GC rewriter lowers to a nursery allocation, so the
+                // first minor collection `run_inner` triggers moves it and
+                // leaves this field naming the vacated block.  Every later
+                // reader takes that address: the vable opcodes below, and the
+                // traceback recorded for each frame an exception propagates
+                // through, which stores it into `PyTraceback.frame` — a slot
+                // the collector does trace, so the stale pointer surfaces as
+                // an invalid type id rather than as a wrong answer.  Root the
+                // SLOT so the walker rewrites it, the same shape
+                // `blackhole_from_resumedata` already applies to the resume
+                // reader's own `virtualizable_ptr` for the chain-build window.
+                majit_gc::shadow_stack::push_resume_ref_roots(std::slice::from_mut(
+                    &mut frame.virtualizable_ptr,
+                ));
                 if !frame.virtualizable_info.is_null() {
                     let vinfo = &*frame.virtualizable_info;
                     vinfo.push_resume_ref_roots_for_registers(&frame.registers_r);

majit/majit-metainterp/src/lib.rs

@@ -154,10 +154,12 @@ pub use pyjitpl::{
     MetaInterpStaticData, RawCompileResult, StandaloneFrameStack, build_state_field_snapshot,
     call_int_function, call_ref_function, call_void_function, counters,
     record_application_traceback_for_recording, record_application_traceback_hook_address,
+    record_discarded_level_traceback_for_recording, record_discarded_level_traceback_hook_address,
     record_inline_application_traceback_for_recording,
     record_inline_application_traceback_hook_address, set_record_application_traceback_hook,
-    set_record_inline_application_traceback_hook, struct_fields_write_effect_info, trace_jitcode,
-    trace_jitcode_from_merge_point, trace_jitcode_with_args, trace_jitcode_with_args_and_runtime,
+    set_record_discarded_level_traceback_hook, set_record_inline_application_traceback_hook,
+    struct_fields_write_effect_info, trace_jitcode, trace_jitcode_from_merge_point,
+    trace_jitcode_with_args, trace_jitcode_with_args_and_runtime,
 };
 pub use resume_box_reader::{
     BridgeVirtualCache, decode_fieldnum, default_bridge_array_descr, emit_pending_field_op,

majit/majit-metainterp/src/pyjitpl.rs

@@ -1571,11 +1571,22 @@ pub struct JitHooks {
 /// majit keeps only this crate-neutral ABI hook.
 pub type RecordApplicationTraceback = extern "C" fn(i64, i64, i64, i64);
 pub type RecordInlineApplicationTraceback = extern "C" fn(i64, i64, i64, i64, i64);
+/// Runtime callback for a frame the resume crossed and then DISCARDED.
+///
+/// `pyopcode.py:148` attaches a node on every frame an unwind passes through,
+/// which upstream gets for free because the unwind is traced code running on a
+/// rebuilt MIFrame stack.  A resume that re-points the live frame straight at
+/// its own handler has no such stack, and the only description left of the
+/// frames it skipped is the `(w_code, python pc)` pair in the resume data —
+/// so this hook takes a PYTHON pc where the two above take a jitcode one.
+pub type RecordDiscardedLevelTraceback = extern "C" fn(i64, i64, i64);
 
 static RECORD_APPLICATION_TRACEBACK: std::sync::atomic::AtomicUsize =
     std::sync::atomic::AtomicUsize::new(0);
 static RECORD_INLINE_APPLICATION_TRACEBACK: std::sync::atomic::AtomicUsize =
     std::sync::atomic::AtomicUsize::new(0);
+static RECORD_DISCARDED_LEVEL_TRACEBACK: std::sync::atomic::AtomicUsize =
+    std::sync::atomic::AtomicUsize::new(0);
 
 pub fn set_record_application_traceback_hook(hook: Option<RecordApplicationTraceback>) {
     RECORD_APPLICATION_TRACEBACK.store(
@@ -1593,6 +1604,13 @@ pub fn set_record_inline_application_traceback_hook(
     );
 }
 
+pub fn set_record_discarded_level_traceback_hook(hook: Option<RecordDiscardedLevelTraceback>) {
+    RECORD_DISCARDED_LEVEL_TRACEBACK.store(
+        hook.map_or(0, |callback| callback as usize),
+        std::sync::atomic::Ordering::Release,
+    );
+}
+
 pub fn record_application_traceback_hook_address() -> *const () {
     RECORD_APPLICATION_TRACEBACK.load(std::sync::atomic::Ordering::Acquire) as *const ()
 }
@@ -1601,6 +1619,10 @@ pub fn record_inline_application_traceback_hook_address() -> *const () {
     RECORD_INLINE_APPLICATION_TRACEBACK.load(std::sync::atomic::Ordering::Acquire) as *const ()
 }
 
+pub fn record_discarded_level_traceback_hook_address() -> *const () {
+    RECORD_DISCARDED_LEVEL_TRACEBACK.load(std::sync::atomic::Ordering::Acquire) as *const ()
+}
+
 fn record_application_traceback(exc_value: i64, frame_ptr: *const u8, frame: &MIFrame) {
     if exc_value == 0 || frame_ptr.is_null() || frame.inline_frame {
         return;
@@ -1667,6 +1689,28 @@ pub fn record_inline_application_traceback_for_recording(
     );
 }
 
+/// Invoke the host callback for a frame the unwind crossed but the resume
+/// discarded, named by its `(w_code, python pc)` coordinate alone.
+///
+/// The two callbacks above both address their frame through a jitcode: one has
+/// the concrete frame pointer, the other the promoted metadata of a level the
+/// walk is still standing in.  A discarded level is neither — the resume data
+/// is the last thing that knows it existed, and what it carries is a Python
+/// coordinate.  Hence the third arity.
+pub fn record_discarded_level_traceback_for_recording(exc_value: i64, w_code: i64, py_pc: i64) {
+    if exc_value == 0 || w_code == 0 {
+        return;
+    }
+    let callback = RECORD_DISCARDED_LEVEL_TRACEBACK.load(std::sync::atomic::Ordering::Acquire);
+    if callback == 0 {
+        return;
+    }
+    // Safety: the only stored values come from a RecordDiscardedLevelTraceback
+    // function pointer in set_record_discarded_level_traceback_hook.
+    let callback: RecordDiscardedLevelTraceback = unsafe { std::mem::transmute(callback) };
+    callback(exc_value, w_code, py_pc);
+}
+
 /// framework.py `root_walker.walk_roots` per-op helper: visit every
 /// inline `ConstPtr.value` slot stored in `op.args` and `op.fail_args`.
 /// history.py:314 `ConstPtr.value` is inline on the Box object; pyre

pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs

@@ -492,6 +492,14 @@ pub fn dispatch_via_miframe<Sym: WalkSym>(
             wc.last_exc_value = Some(value_op);
             wc.last_exc_value_concrete = ConcreteValue::Ref(exc_edge_concrete);
             wc.fbw_mode.class_of_last_exc_is_const = true;
+            // The inlined callees this route unwound clear out of, innermost
+            // first, BEFORE the catching frame's own node: both recorders
+            // prepend, so emission order is the chain read outermost-first.
+            record_exc_edge_discarded_tracebacks(
+                &mut wc,
+                value_op,
+                ConcreteValue::Ref(exc_edge_concrete),
+            );
             record_bridge_handler_entry_traceback(
                 &mut wc,
                 value_op,

pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs

@@ -562,6 +562,61 @@ fn record_top_level_application_traceback<Sym: WalkSym>(
     }
 }
 
+/// One node per inlined level the exception-edge bridge resumed PAST, for the
+/// frames `set_exc_edge_discarded_levels` parked before the walk began.
+///
+/// `pyopcode.py:148 pytraceback.record_application_traceback` runs before the
+/// `:152` exception-table lookup, so a frame the unwind only PASSES THROUGH
+/// contributes a node just like the one that catches it.  Upstream never has to
+/// say so: it resumes onto a rebuilt MIFrame stack and the unwind is traced code
+/// that runs each level's own recorder.  The exc-edge route exists precisely
+/// because pyre discards those levels instead, which leaves this the only place
+/// their nodes can come from.
+///
+/// Innermost-first, because `record_application_traceback` PREPENDS: the deepest
+/// discarded frame must go on while the node the residual raise already attached
+/// is still the head.  The caller then records the catching frame last, so the
+/// chain reads outermost-first exactly as the interpreter builds it.
+///
+/// Emitted as IR, not merely executed: `trace_and_compile_from_bridge` runs once,
+/// and every later guard failure of this class enters the compiled bridge
+/// directly.
+fn record_exc_edge_discarded_tracebacks<Sym: WalkSym>(
+    ctx: &mut WalkContext<'_, '_, Sym>,
+    exc: OpRef,
+    exc_concrete: ConcreteValue,
+) {
+    let levels = take_exc_edge_discarded_levels();
+    if levels.is_empty() {
+        return;
+    }
+    let ConcreteValue::Ref(exc_ptr) = exc_concrete else {
+        return;
+    };
+    if exc_ptr.is_null() {
+        return;
+    }
+    let hook = majit_metainterp::record_discarded_level_traceback_hook_address();
+    for &(w_code, py_pc) in levels.iter().rev() {
+        majit_metainterp::record_discarded_level_traceback_for_recording(
+            exc_ptr as usize as i64,
+            w_code as i64,
+            py_pc as i64,
+        );
+        if hook.is_null() || exc.is_none() {
+            continue;
+        }
+        let w_code_op = ctx.trace_ctx.const_ref(w_code as i64);
+        let py_pc_op = ctx.trace_ctx.const_int(py_pc as i64);
+        ctx.trace_ctx.call_void_typed_with_effect(
+            hook,
+            &[exc, w_code_op, py_pc_op],
+            &[Type::Ref, Type::Ref, Type::Int],
+            default_effect_info(),
+        );
+    }
+}
+
 fn record_inline_application_traceback<Sym: WalkSym>(
     ctx: &mut WalkContext<'_, '_, Sym>,
     exc: OpRef,
@@ -2306,6 +2361,35 @@ pub(crate) fn take_carrier_raise_seed() -> Option<CarrierRaiseSeed> {
     FBW_CARRIER_RAISE_SEED.with(|c| c.take())
 }
 
+thread_local! {
+    /// The inlined levels an exception-edge bridge resumes PAST, outermost-first
+    /// and excluding the live frame, as `(w_code, python pc)`.
+    ///
+    /// `call_jit.rs trace_and_compile_from_bridge` routes the exc edge only for
+    /// a raise that unwinds clear out of every inlined callee, and answers that
+    /// question from the per-level `resume_coords` it decoded.  Those coordinates
+    /// are the last description of the discarded levels anywhere in the process —
+    /// the walk that follows starts flat — so they are parked here for the walk's
+    /// handler entry to turn into traceback nodes.
+    static FBW_EXC_EDGE_DISCARDED_LEVELS: std::cell::RefCell<Vec<(usize, usize)>> =
+        const { std::cell::RefCell::new(Vec::new()) };
+}
+
+/// Publish the levels an exc-edge route is about to discard.  Always called on
+/// the routing path, with an empty slice for a single-frame resume, so a
+/// previous bridge's levels can never leak into this one.
+pub fn set_exc_edge_discarded_levels(levels: &[(usize, usize)]) {
+    FBW_EXC_EDGE_DISCARDED_LEVELS.with(|c| {
+        let mut slot = c.borrow_mut();
+        slot.clear();
+        slot.extend_from_slice(levels);
+    });
+}
+
+pub(crate) fn take_exc_edge_discarded_levels() -> Vec<(usize, usize)> {
+    FBW_EXC_EDGE_DISCARDED_LEVELS.with(|c| std::mem::take(&mut *c.borrow_mut()))
+}
+
 /// Walk one opcode at `pc` and return the dispatch outcome plus the
 /// next pc. Side effects reach `ctx.trace_ctx` only for opnames whose
 /// handler explicitly records (e.g. `ref_return/r` calls

pyre/pyre-jit-trace/src/state.rs

@@ -4449,14 +4449,29 @@ pub(crate) fn record_quasiimmut_field(ctx: &mut TraceCtx, obj: OpRef, descr: Des
         );
         return;
     }
+    // quasiimmut.py:124 `self.qmut = get_current_qmut_instance(cpu, struct,
+    // mutatefielddescr)` — the half that makes the value captured below
+    // answerable later.  The hidden `mutate_<name>` field is null until
+    // something installs it, and while it is null a write to the
+    // quasi-immutable field takes the no-watchers early return: the value moves
+    // and nothing is forced.  Installing at the record makes every write from
+    // here on run the invalidation, which is what both the tracer's own force
+    // check and the optimizer's revalidation read.
+    //
+    // Ordered before the value read for the reason upstream orders `__init__`
+    // that way: a write landing between the two must be one the watcher sees.
+    // Reading first leaves a window where the field moves with no watcher
+    // installed, so nothing invalidates and nothing bumps the force counter,
+    // and the trace keeps a value that is already stale.  Without a GIL that
+    // window is a real interleaving, not a theoretical one.
+    install_quasiimmut_field(ctx, obj, &descr);
     // quasiimmut.py:125 `self.constantfieldbox =
     // self.get_current_constant_fieldvalue()` — the field's value at the moment
     // the trace baked it.  `heap.py:803 is_still_valid_for` compares it against
     // the live value and abandons the loop when they disagree, so it has to be
     // captured here; by the time the optimizer runs, the change it is looking
     // for has already happened.
     let constantfieldbox = current_quasiimmut_field_value(ctx, obj, &descr);
-    install_quasiimmut_field(ctx, obj, &descr);
     ctx.heap_cache_mut().quasi_immut_now_known(field_index, obj);
     // Upstream carries the captured value on the per-op `QuasiImmutDescr`
     // (quasiimmut.py:113-159), a descr minted fresh for every recorded

pyre/pyre-jit-trace/src/trace.rs

@@ -3976,7 +3976,24 @@ fn run_perfn_walk<Sym: WalkSym>(
         }) = &walk_result
         {
             let abort_jit_pc = *pc;
-            if !crate::jitcode_dispatch::fbw_executed_nonpure_residual() {
+            // A walk commits at most ONE leg.  The carrier block above may
+            // already have taken `CalleeRebuild`, which resumes INSIDE the
+            // rebuilt callee, past what it applied; rewinding the caller to its
+            // CALL on top of that runs the whole callee a second time.  Nothing
+            // else keeps the two apart — this leg's own gates are inclusion
+            // tests on the residual odometer, and `walk_end_resume_provable`
+            // cannot see effects applied by the plain interpretation the
+            // rebuild resumed into (the `MidBodyDecline::AfterRun` argument,
+            // which the carrier block applies only to its own fallback).
+            if WALK_END_FLUSH_COMMITTED.with(|c| c.get()) {
+                crate::jitcode_dispatch::fbw_abort_outer_resume_reset();
+                if crate::jitcode_dispatch::fbw_debug_abort_enabled() {
+                    eprintln!(
+                        "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \
+                             (a carrier leg already committed this walk)"
+                    );
+                }
+            } else if !crate::jitcode_dispatch::fbw_executed_nonpure_residual() {
                 if crate::jitcode_dispatch::fbw_debug_abort_enabled() {
                     eprintln!(
                         "[fbw-abort-flush] declined at abort_jit_pc={abort_jit_pc} \

pyre/pyre-jit/src/call_jit.rs

@@ -807,6 +807,84 @@ pub(crate) extern "C" fn record_inline_traceback_for_recording(
     }
 }
 
+/// `pytraceback.py:104-109 record_application_traceback` for a frame the resume
+/// crossed and discarded — an inlined callee the exception-edge bridge unwound
+/// clear out of on its way to the live frame's own handler.
+///
+/// `pyopcode.py:148` runs BEFORE the `:152` exception-table lookup, so a frame
+/// that only propagates contributes a node exactly like one that catches.  The
+/// levels named here never reach a recorder that could speak for them: the
+/// route exists because the unwind discards them, and by the time the walk
+/// starts they are gone.
+///
+/// The node's frame is fabricated from the code object, as
+/// [`record_inline_traceback_for_recording`] does and for the same reason —
+/// the level's own `PyFrame` stayed virtual in the compiled trace and the
+/// resume never materialized it, so there is no other object to name.  Unlike
+/// that case there is also no seeded frame it could be confused with: nothing
+/// else in this process can reach the discarded level either.
+///
+/// `py_pc` is the level's resume coordinate, which is `next_instr`-style — the
+/// same coordinate `exc_table_offset` converts with `saturating_sub(1)` and
+/// `set_last_instr_from_next_instr` converts for the live frame.  Everything
+/// below wants the instruction that RAN: the opcode decode that answers the
+/// bare-reraise question, `frame.last_instr`, and `record_application_traceback`'s
+/// `tb_lasti`.  Convert once, up front.
+pub(crate) extern "C" fn record_discarded_level_traceback(
+    exc_value: i64,
+    w_code_value: i64,
+    py_pc: i64,
+) {
+    if exc_value == 0 || w_code_value == 0 || py_pc < 0 {
+        return;
+    }
+    let last_instruction = py_pc.saturating_sub(1);
+    let w_code = w_code_value as PyObjectRef;
+    let raw_code =
+        unsafe { pyre_interpreter::w_code_get_ptr(w_code) as *const pyre_interpreter::CodeObject };
+    if raw_code.is_null() {
+        return;
+    }
+    // Same RaiseWithExplicitTraceback rule the other two recorders follow: a
+    // bare reraise preserves the traceback the original raise attached.
+    let bare_reraise = match unsafe {
+        pyre_interpreter::decode_instruction_at(&*raw_code, last_instruction as usize)
+    } {
+        Some((pyre_interpreter::Instruction::RaiseVarargs { .. }, op_arg)) => {
+            u32::from(op_arg) == 0
+        }
+        Some((pyre_interpreter::Instruction::Reraise { .. }, _)) => true,
+        _ => false,
+    };
+    if bare_reraise {
+        return;
+    }
+    let w_globals = unsafe { pyre_interpreter::w_code_get_w_globals(w_code) };
+    let w_exc = exc_value as PyObjectRef;
+    let _roots = pyre_object::gc_roots::push_roots();
+    pyre_object::gc_roots::pin_root(w_exc);
+    pyre_object::gc_roots::pin_root(w_code);
+    pyre_object::gc_roots::pin_root(w_globals);
+    let Ok(mut frame) = pyre_interpreter::createframe_obj(
+        w_code as *const (),
+        w_globals,
+        pyre_interpreter::call::getexecutioncontext(),
+        None,
+    ) else {
+        return;
+    };
+    frame.last_instr = last_instruction as isize;
+    let frame_ptr = frame.into_raw();
+    pyre_object::gc_roots::pin_root(frame_ptr as PyObjectRef);
+    unsafe {
+        pyre_interpreter::pytraceback::record_application_traceback(
+            w_exc,
+            frame_ptr,
+            last_instruction,
+        );
+    }
+}
+
 #[majit_macros::jit_may_force]
 pub extern "C" fn jit_force_callee_frame(frame_ptr: i64) -> i64 {
     #[cfg(feature = "cranelift")]
@@ -3328,6 +3406,15 @@ pub fn trace_and_compile_from_bridge(
     let route_exc_edge = caught_in_frame
         && (!is_multiframe_resume || unwind_to_live_frame)
         && pyre_jit_trace::jitcode_dispatch::exc_edge_bridge_enabled();
+    // The levels this route is about to throw away: `resume_coords` minus the
+    // live frame, which keeps its own recorder at the handler entry.  Published
+    // unconditionally on the routing path — an empty slice for the single-frame
+    // shape — so a previous bridge's levels cannot survive into this walk.
+    pyre_jit_trace::jitcode_dispatch::set_exc_edge_discarded_levels(if route_exc_edge {
+        resume_coords.get(1..).unwrap_or(&[])
+    } else {
+        &[]
+    });
     if route_exc_edge && guard_exc != 0 {
         // Publish the grabbed exception (`cpu.grab_exc_value` result) so the
         // walker's `seed_standing_exception_for_walk` threads it into