From 53e6e567cf513a0458c4dcb05ea9634741eef324 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Jul 2026 20:40:20 +0900 Subject: [PATCH 1/8] jit: carry forward per-frame py_pc through resume data + audit twin (jitcode-blackhole Slice 3' Phase A) Add a per-frame Python-pc word to the resume-data frame header, sourced forward from the codewriter `(jitcode_pc, py_pc)` marker pair at guard capture, rather than derived backward via `backxlat_py_pc` at decode. Thread `py_pc` through `recorder::SnapshotFrame`, `resume::SnapshotFrame`, `FrameInfoBuilder`/`push_frame`, and `RebuiltFrame`; write it in all three frame-header encoders (`number()` and the two compact `rd_numb` encoders) and consume it in both decoders (`rebuild_from_numbering` and the blackhole `read_jitcode_pos_pc`). Wire order is `jitcode_index, pc, py_pc, [boxes]` uniformly. Synthetic single frames use the `-1` sentinel. `build_resumed_frames` still derives the value via `backxlat_py_pc`; under `PYRE_M73_PYPC_FWD_AUDIT` it asserts the forward word equals the backxlat result for every frame. Off by default, so production decode is byte-identical. Consumer cutover + backxlat-call-site deletion is Phase B. Remove the dead `ResumedFrame.rd_numb_pc` field (no readers repo-wide). check.py: dynasm 248/248 + cranelift 248/248 with the audit off (byte-identical) and with `PYRE_M73_PYPC_FWD_AUDIT=1` (zero divergence). Assisted-by: Claude --- majit/majit-backend/src/resume_value.rs | 3 + majit/majit-ir/src/resumedata.rs | 15 ++++- majit/majit-metainterp/src/compile.rs | 5 +- majit/majit-metainterp/src/history.rs | 19 ++++-- majit/majit-metainterp/src/optimizeopt/mod.rs | 32 +++++++--- .../src/optimizeopt/optimizer.rs | 2 +- .../src/optimizeopt/unroll.rs | 2 +- majit/majit-metainterp/src/pyjitpl.rs | 4 +- .../majit-metainterp/src/pyjitpl/dispatch.rs | 1 + majit/majit-metainterp/src/recorder.rs | 3 + majit/majit-metainterp/src/resume.rs | 59 ++++++++++++++----- .../tests/jit_driver_runtime_parity.rs | 25 ++++---- majit/majit-metainterp/tests/resume_parity.rs | 20 +++++-- .../src/jitcode_dispatch/diag.rs | 8 +++ .../src/jitcode_dispatch/resume_snapshot.rs | 28 ++++++++- pyre/pyre-jit-trace/src/pyjitcode.rs | 33 +++++++++++ pyre/pyre-jit-trace/src/state.rs | 3 + pyre/pyre-jit-trace/src/trace_opcode.rs | 5 ++ pyre/pyre-jit/src/call_jit.rs | 1 - pyre/pyre-jit/src/eval.rs | 11 +++- pyre/pyre-jit/src/jit/codewriter.rs | 10 ++++ 21 files changed, 231 insertions(+), 58 deletions(-) diff --git a/majit/majit-backend/src/resume_value.rs b/majit/majit-backend/src/resume_value.rs index af0d2433408..3baf9f4b852 100644 --- a/majit/majit-backend/src/resume_value.rs +++ b/majit/majit-backend/src/resume_value.rs @@ -174,6 +174,9 @@ pub struct FrameInfo { /// populates it with the Python bytecode PC because pyre's tracer /// records Python bytecode rather than JitCode. pub pc: u64, + /// Forward-carried Python instruction PC. `-1` is the no-snapshot + /// sentinel and must round-trip through every resume encoder unchanged. + pub py_pc: i32, /// Mapping from slot index to a tagged resume source. pub slot_map: Vec, } diff --git a/majit/majit-ir/src/resumedata.rs b/majit/majit-ir/src/resumedata.rs index b554b82aa9a..48d0a20b668 100644 --- a/majit/majit-ir/src/resumedata.rs +++ b/majit/majit-ir/src/resumedata.rs @@ -355,6 +355,9 @@ pub struct RebuiltFrame { pub jitcode_index: i32, /// resume.py:250 `pc` — the JitCode byte offset. pub pc: i32, + /// Forward-carried Python instruction PC; `-1` is the no-snapshot + /// sentinel paired with `pc == -1`. + pub py_pc: i32, pub values: Vec, } @@ -430,14 +433,14 @@ pub fn decode_tagged_value( /// Decode rd_numb back into vable/vref values and per-frame tagged values. /// /// resume.py:249-253, resume.py:1049-1055: RPython encodes frames as -/// `jitcode_index, pc, [tagged_values...]` and uses jitcode liveness +/// `jitcode_index, pc, py_pc, [tagged_values...]` and uses jitcode liveness /// (`get_current_position_info`) at the decode site to know how many /// values each frame has. /// /// `frame_value_count`: when `Some(f)`, `f(jitcode_index, pc)` /// returns the number of tagged values for that frame (RPython parity: /// liveness-driven decode). When `None`, all remaining items after -/// `(jitcode_index, pc)` are consumed as a single frame (backward-compat for +/// `(jitcode_index, pc, py_pc)` are consumed as a single frame (backward-compat for /// callers that only ever see single-frame data). /// /// `fail_arg_types`: parent guard's per-failarg type vector. resume.py:1245 @@ -490,7 +493,7 @@ pub fn rebuild_from_numbering( )); } - // resume.py:1049-1055: frame section — jitcode_index, pc, [tagged_values...]. + // Pyre M73: frame section — jitcode_index, pc, py_pc, [tagged_values...]. // RPython uses consume_one_section → enumerate_vars(liveness) to split frames. let mut frames = Vec::new(); while reader.items_read < total_size as usize && reader.has_more() { @@ -500,6 +503,11 @@ pub fn rebuild_from_numbering( } else { 0 }; + let py_pc = if reader.has_more() && reader.items_read < total_size as usize { + reader.next_item() + } else { + -1 + }; let box_count = if let Some(f) = &frame_value_count { // RPython parity: liveness-driven frame boundary. f(jitcode_index, pc) @@ -524,6 +532,7 @@ pub fn rebuild_from_numbering( frames.push(RebuiltFrame { jitcode_index, pc, + py_pc, values, }); } diff --git a/majit/majit-metainterp/src/compile.rs b/majit/majit-metainterp/src/compile.rs index 61c8e72de25..44b9873f8c5 100644 --- a/majit/majit-metainterp/src/compile.rs +++ b/majit/majit-metainterp/src/compile.rs @@ -586,7 +586,7 @@ pub(crate) fn build_guard_metadata>( // as separate sections. Do not merge vable_array entries into // the innermost frame slots here. for frame in frames.iter() { - builder.push_frame(frame.jitcode_index, frame.pc as u64); + builder.push_frame(frame.jitcode_index, frame.pc as u64, frame.py_pc); let mut slot_idx = 0usize; for val in &frame.values { add_slot(&mut builder, slot_idx, val); @@ -595,7 +595,7 @@ pub(crate) fn build_guard_metadata>( } } else { // No rd_numb: single frame, 1:1 mapping (fail_args[i] → state[i]). - builder.push_frame(0, pc); + builder.push_frame(0, pc, -1); let num_slots = op .getfailargs() .map(|fa| fa.len()) @@ -2625,6 +2625,7 @@ mod tests { framestack: vec![SnapshotFrame { jitcode_index: 0, pc: 8, + py_pc: 8, boxes: vec![OpRef::input_arg_int(1).into()], }], }; diff --git a/majit/majit-metainterp/src/history.rs b/majit/majit-metainterp/src/history.rs index f010986ffb3..e00acbd6569 100644 --- a/majit/majit-metainterp/src/history.rs +++ b/majit/majit-metainterp/src/history.rs @@ -1014,6 +1014,7 @@ impl TreeLoop { .map(|f| crate::recorder::SnapshotFrame { jitcode_index: f.jitcode_index, pc: f.pc, + py_pc: f.py_pc, boxes: f.boxes.iter().map(&remap_tagged).collect(), }) .collect(), @@ -2229,6 +2230,7 @@ impl TraceCtx { active_boxes, jitcode_index, pc, + pc, &[], &[], ); @@ -2254,6 +2256,7 @@ impl TraceCtx { active_boxes: &[OpRef], jitcode_index: u32, pc: u32, + py_pc: u32, vable_boxes: &[crate::recorder::SnapshotTagged], vref_boxes: &[crate::recorder::SnapshotTagged], ) { @@ -2263,6 +2266,7 @@ impl TraceCtx { frames: vec![crate::recorder::SnapshotFrame { jitcode_index, pc, + py_pc, boxes, }], vable_boxes: vable_boxes.to_vec(), @@ -2281,6 +2285,7 @@ impl TraceCtx { active_boxes: &[OpRef], jitcode_index: u32, pc: u32, + py_pc: u32, vable_boxes: &[crate::recorder::SnapshotTagged], vref_boxes: &[crate::recorder::SnapshotTagged], ) { @@ -2289,6 +2294,7 @@ impl TraceCtx { frames: vec![crate::recorder::SnapshotFrame { jitcode_index, pc, + py_pc, boxes, }], vable_boxes: vable_boxes.to_vec(), @@ -2314,7 +2320,7 @@ impl TraceCtx { /// (RPython's `_number_boxes` does this implicitly via the memo /// table; pyre's `Snapshot.encode` does the same in /// `resume.rs:1898 _number_boxes`). - pub fn capture_snapshot_for_last_guard_multi_frame(&mut self, frames: &[(u32, u32, &[OpRef])]) { + pub fn capture_snapshot_for_last_guard_multi_frame(&mut self, frames: &[(u32, u32, u32, &[OpRef])]) { self.capture_snapshot_for_last_guard_multi_frame_with_vable_vref(frames, &[], &[]); } @@ -2328,18 +2334,19 @@ impl TraceCtx { /// it sees in the trace-time MIFrame stack. pub fn capture_snapshot_for_last_guard_multi_frame_with_vable_vref( &mut self, - frames: &[(u32, u32, &[OpRef])], + frames: &[(u32, u32, u32, &[OpRef])], vable_boxes: &[crate::recorder::SnapshotTagged], vref_boxes: &[crate::recorder::SnapshotTagged], ) { let recorder_frames: Vec = frames .iter() - .map(|(jitcode_index, pc, boxes)| { + .map(|(jitcode_index, pc, py_pc, boxes)| { // The pc word is a raw JitCode offset. let encoded = self.encode_snapshot_boxes(boxes); crate::recorder::SnapshotFrame { jitcode_index: *jitcode_index, pc: *pc, + py_pc: *py_pc, boxes: encoded, } }) @@ -2362,18 +2369,19 @@ impl TraceCtx { /// paused caller chain is available for a full `Snapshot.frames`. pub fn capture_snapshot_for_last_guard_op_multi_frame_with_vable_vref( &mut self, - frames: &[(u32, u32, &[OpRef])], + frames: &[(u32, u32, u32, &[OpRef])], vable_boxes: &[crate::recorder::SnapshotTagged], vref_boxes: &[crate::recorder::SnapshotTagged], ) { let recorder_frames: Vec = frames .iter() - .map(|(jitcode_index, pc, boxes)| { + .map(|(jitcode_index, pc, py_pc, boxes)| { // The pc word is a raw JitCode offset. let encoded = self.encode_snapshot_boxes(boxes); crate::recorder::SnapshotFrame { jitcode_index: *jitcode_index, pc: *pc, + py_pc: *py_pc, boxes: encoded, } }) @@ -2842,6 +2850,7 @@ impl TraceCtx { frames: vec![crate::recorder::SnapshotFrame { jitcode_index: 0, pc: self.last_traced_pc as u32, + py_pc: self.last_traced_pc as u32, boxes: Vec::new(), }], vable_boxes: Vec::new(), diff --git a/majit/majit-metainterp/src/optimizeopt/mod.rs b/majit/majit-metainterp/src/optimizeopt/mod.rs index 6e80576c6b3..08b39777afd 100644 --- a/majit/majit-metainterp/src/optimizeopt/mod.rs +++ b/majit/majit-metainterp/src/optimizeopt/mod.rs @@ -45,7 +45,7 @@ use std::collections::VecDeque; pub type SnapshotBoxes = Vec>>; pub type SnapshotFrameSizes = Vec>>; -pub type SnapshotFramePcs = Vec>>; +pub type SnapshotFramePcs = Vec>>; type OpRefFxIndexMap = indexmap::IndexMap; pub(crate) fn snapshot_get(store: &[Option], pos: i32) -> Option<&T> { @@ -754,7 +754,7 @@ pub struct OptContext { /// resume.py:243-247 _number_boxes consumes vref_array as a section /// after vable_array. opencoder.py:767 records vref_boxes here. pub snapshot_vref_boxes: SnapshotBoxes, - /// Per-guard per-frame (jitcode_index, pc) from tracing-time snapshots. + /// Per-guard per-frame (jitcode_index, pc, py_pc) from tracing-time snapshots. pub snapshot_frame_pcs: SnapshotFramePcs, /// optimizer.py:34 `self.inputargs = inputargs` parity. /// Typed InputArg OpRefs; slot `i` is `OpRef::input_arg_typed(i, tp)`. @@ -6244,14 +6244,32 @@ impl OptContext { for (i, &size) in sizes.iter().enumerate() { let end = (offset + size).min(snapshot_boxes.len()); let frame_boxes: Vec = snapshot_boxes[offset..end].to_vec(); - let (jitcode_index, pc) = frame_pcs.get(i).copied().unwrap_or((0, 0)); - frames.push((jitcode_index, pc, frame_boxes)); + let (jitcode_index, pc, py_pc) = frame_pcs.get(i).copied().unwrap_or((0, 0, 0)); + frames.push(crate::resume::SnapshotFrame { + jitcode_index, + pc, + py_pc, + boxes: frame_boxes, + }); offset = end; } - Snapshot::multi_frame_boxes(frames) + Snapshot { + vable_array: Vec::new(), + vref_array: Vec::new(), + framestack: frames, + } } else { - let (jitcode_index, pc) = frame_pcs.first().copied().unwrap_or((0, 0)); - Snapshot::single_frame_boxes(jitcode_index, pc, snapshot_boxes.clone()) + let (jitcode_index, pc, py_pc) = frame_pcs.first().copied().unwrap_or((0, 0, 0)); + Snapshot { + vable_array: Vec::new(), + vref_array: Vec::new(), + framestack: vec![crate::resume::SnapshotFrame { + jitcode_index, + pc, + py_pc, + boxes: snapshot_boxes.clone(), + }], + } }; // pyjitpl.py:2588: vable_array stores virtualizable_boxes. // ni/vsd are constants (TAGINT/TAGCONST) so they don't affect diff --git a/majit/majit-metainterp/src/optimizeopt/optimizer.rs b/majit/majit-metainterp/src/optimizeopt/optimizer.rs index 3b7c4afdf77..e145ec76e15 100644 --- a/majit/majit-metainterp/src/optimizeopt/optimizer.rs +++ b/majit/majit-metainterp/src/optimizeopt/optimizer.rs @@ -438,7 +438,7 @@ pub struct Optimizer { /// section. opencoder.py:767 create_top_snapshot records vref_boxes /// alongside vable_boxes. pub snapshot_vref_boxes: SnapshotBoxes, - /// Per-guard per-frame (jitcode_index, pc) from tracing-time snapshots. + /// Per-guard per-frame (jitcode_index, pc, py_pc) from tracing-time snapshots. pub snapshot_frame_pcs: SnapshotFramePcs, /// Phase 1 emit ops carried into Phase 2's lookup surface (6). /// diff --git a/majit/majit-metainterp/src/optimizeopt/unroll.rs b/majit/majit-metainterp/src/optimizeopt/unroll.rs index 1e30e142d32..22b48e2dbee 100644 --- a/majit/majit-metainterp/src/optimizeopt/unroll.rs +++ b/majit/majit-metainterp/src/optimizeopt/unroll.rs @@ -294,7 +294,7 @@ pub struct UnrollOptimizer { /// (resume.py:243-247 vref_array — _number_boxes consumes them /// after the virtualizable array). pub snapshot_vref_boxes: SnapshotBoxes, - /// Per-guard per-frame (jitcode_index, pc) from tracing-time snapshots. + /// Per-guard per-frame (jitcode_index, pc, py_pc) from tracing-time snapshots. pub snapshot_frame_pcs: SnapshotFramePcs, /// pyjitpl.py:2289 all_descrs: dense list indexed by descr_index. /// Threaded through inner Optimizer instances for inline registration. diff --git a/majit/majit-metainterp/src/pyjitpl.rs b/majit/majit-metainterp/src/pyjitpl.rs index 6366f582288..23748437256 100644 --- a/majit/majit-metainterp/src/pyjitpl.rs +++ b/majit/majit-metainterp/src/pyjitpl.rs @@ -552,10 +552,10 @@ fn snapshot_map_from_trace_snapshots( // AND vref_array. resume.py:243-247 _number_boxes consumes // vref_array as a separate section after vable_array. let vref_boxes: Vec = snap.vref_boxes.iter().map(&tagged_to_box).collect(); - let frame_pcs: Vec<(i32, i32)> = snap + let frame_pcs: Vec<(i32, i32, i32)> = snap .frames .iter() - .map(|f| (f.jitcode_index as i32, f.pc as i32)) + .map(|f| (f.jitcode_index as i32, f.pc as i32, f.py_pc as i32)) .collect(); let id = id as i32; snapshot_insert(&mut box_map, id, boxes); diff --git a/majit/majit-metainterp/src/pyjitpl/dispatch.rs b/majit/majit-metainterp/src/pyjitpl/dispatch.rs index b3e0b1b88a8..2a25b271d92 100644 --- a/majit/majit-metainterp/src/pyjitpl/dispatch.rs +++ b/majit/majit-metainterp/src/pyjitpl/dispatch.rs @@ -8036,6 +8036,7 @@ pub fn build_state_field_snapshot( snapshot_frames.push(crate::recorder::SnapshotFrame { jitcode_index, pc: frame.pc as u32, + py_pc: frame.pc as u32, boxes, }); } diff --git a/majit/majit-metainterp/src/recorder.rs b/majit/majit-metainterp/src/recorder.rs index 9195353a8ee..1f53f7a5bc0 100644 --- a/majit/majit-metainterp/src/recorder.rs +++ b/majit/majit-metainterp/src/recorder.rs @@ -74,6 +74,9 @@ pub struct SnapshotFrame { /// context; the runtime translates `py_pc` through `pc_map` at resume /// time until pyre's walker-as-tracer epic lands. pub pc: u32, + /// Forward-carried Python instruction PC for this JitCode position. + /// `u32::MAX` is the no-snapshot sentinel paired with `pc == -1`. + pub py_pc: u32, /// Tagged references to the live boxes in this frame. pub boxes: Vec, } diff --git a/majit/majit-metainterp/src/resume.rs b/majit/majit-metainterp/src/resume.rs index c8d48309d5e..2ab3672c2a6 100644 --- a/majit/majit-metainterp/src/resume.rs +++ b/majit/majit-metainterp/src/resume.rs @@ -273,6 +273,9 @@ pub struct SnapshotFrame { pub jitcode_index: i32, /// JitCode byte offset (resume.py:250 pc). pub pc: i32, + /// Forward-carried Python instruction PC. `-1` is the no-snapshot + /// sentinel, paired with `pc == -1`. + pub py_pc: i32, /// Live boxes for this frame's registers (resume.py:253). pub boxes: Vec, } @@ -300,6 +303,7 @@ impl Snapshot { framestack: vec![SnapshotFrame { jitcode_index, pc, + py_pc: pc, boxes, }], } @@ -337,6 +341,7 @@ impl Snapshot { .map(|(jitcode_index, pc, boxes)| SnapshotFrame { jitcode_index, pc, + py_pc: pc, boxes, }) .collect(), @@ -345,7 +350,7 @@ impl Snapshot { /// Estimated encoded size for NumberingState capacity hint. pub fn estimated_size(&self) -> usize { - let frame_size: usize = self.framestack.iter().map(|f| f.boxes.len() + 2).sum(); + let frame_size: usize = self.framestack.iter().map(|f| f.boxes.len() + 3).sum(); self.vable_array.len() + self.vref_array.len() + frame_size + 4 } } @@ -696,6 +701,7 @@ fn resume_frame_layout_to_frame_info(layout: &ResumeFrameLayoutSummary) -> Frame FrameInfo { jitcode_index: layout.jitcode_index, pc: layout.pc, + py_pc: -1, slot_map: layout .slot_layouts .iter() @@ -1975,6 +1981,7 @@ impl EncodedResumeData { for frame in frames { rd_numb.push(frame.jitcode_index as i64); rd_numb.push(encode_u64(frame.pc)); + rd_numb.push(encode_u64(frame.py_pc as u64)); // resume.py:253 _number_boxes(snapshot_iter, iter_array(snapshot), numb_state) for source in &frame.slot_map { let tagged = memo.encode_tagged_source(source, &mut liveboxes, &mut box_map); @@ -2052,7 +2059,7 @@ impl EncodedResumeData { vref_array.push(self.decode_box(self.next_word(&mut cursor))); } // resume.py:1049-1055: frame section. - // Per-frame: jitcode_index, pc, [tagged_values...]. + // Per-frame: jitcode_index, pc, py_pc, [tagged_values...]. // RPython uses jitcode.get_live_vars_info(pc) for frame boundary; // we use self.frame_sizes[] stored at encode time. let items_resume_len = decode_len(items_resume_section); @@ -2061,6 +2068,7 @@ impl EncodedResumeData { while cursor < items_resume_len { let jitcode_index = self.next_word(&mut cursor) as i32; let pc = decode_u64(self.next_word(&mut cursor)); + let py_pc = self.next_word(&mut cursor) as i32; let slot_count = if frame_idx < self.frame_sizes.len() { self.frame_sizes[frame_idx] } else { @@ -2074,6 +2082,7 @@ impl EncodedResumeData { frames.push(FrameInfo { jitcode_index, pc, + py_pc, slot_map, }); frame_idx += 1; @@ -2342,6 +2351,7 @@ impl ResumeDataExt for ResumeData { frames: vec![FrameInfo { jitcode_index: 0, pc, + py_pc: -1, slot_map, }], virtuals: Vec::new(), @@ -2877,6 +2887,7 @@ pub struct ResumeDataVirtualAdder { struct FrameInfoBuilder { jitcode_index: i32, pc: u64, + py_pc: i32, slot_map: Vec, } @@ -2897,11 +2908,12 @@ impl ResumeDataVirtualAdder { } /// Push a new frame onto the stack. - /// resume.py:249-252: jitcode_index, pc per frame. - pub fn push_frame(&mut self, jitcode_index: i32, pc: u64) { + /// resume.py:249-252: jitcode_index, pc, py_pc per frame. + pub fn push_frame(&mut self, jitcode_index: i32, pc: u64, py_pc: i32) { self.frames.push(FrameInfoBuilder { jitcode_index, pc, + py_pc, slot_map: Vec::new(), }); } @@ -3082,6 +3094,7 @@ impl ResumeDataVirtualAdder { .map(|f| FrameInfo { jitcode_index: f.jitcode_index, pc: f.pc, + py_pc: f.py_pc, slot_map: f.slot_map, }) .collect(), @@ -3841,12 +3854,12 @@ impl ResumeDataLoopMemo { /// [tagged boxes for vable_array] /// [n] vref_array_length (0 if no virtualrefs) /// [tagged boxes for vref_array] - /// [m] frame0_pc frame0_slots... - /// [m+] frame1_pc frame1_slots... + /// [m] frame0_jitcode_index frame0_pc frame0_py_pc frame0_slots... + /// [m+] frame1_jitcode_index frame1_pc frame1_py_pc frame1_slots... /// ... /// ``` /// - /// `frames` is a list of (pc, fail_args_slice) for each frame. + /// `frames` carries `(jitcode_index, pc, py_pc, fail_args_slice)` for each frame. /// In pyre (single frame), this is typically one frame. /// resume.py:228-256 number() — serialize a guard's full snapshot. /// @@ -3907,12 +3920,13 @@ impl ResumeDataLoopMemo { self._number_boxes(&snapshot.vref_array, &mut numb_state, env)?; // resume.py:249-253: frame chain. - // Per-frame: jitcode_index, pc, [tagged_values...]. + // Per-frame: jitcode_index, pc, py_pc, [tagged_values...]. // RPython uses jitcode.get_live_vars_info(pc) at decode time // to know how many tagged values each frame has. for frame in &snapshot.framestack { numb_state.append_int(frame.jitcode_index as i64); numb_state.append_int(frame.pc as i64); + numb_state.append_int(frame.py_pc as i64); self._number_boxes(&frame.boxes, &mut numb_state, env)?; } @@ -4246,11 +4260,12 @@ impl ResumeDataLoopMemo { rd_numb.push(tagged); } - // resume.py:249-253: per-frame: jitcode_index, pc, [tagged_values...]. + // resume.py:249-253: per-frame: jitcode_index, pc, py_pc, [tagged_values...]. let mut frame_sizes = Vec::with_capacity(rd.frames.len()); for frame in &rd.frames { rd_numb.push(frame.jitcode_index as i64); rd_numb.push(encode_u64(frame.pc)); + rd_numb.push(encode_u64(frame.py_pc as u64)); for source in &frame.slot_map { let tagged = self.encode_tagged_source(source, &mut liveboxes, &mut box_map); rd_numb.push(tagged); @@ -4573,6 +4588,7 @@ mod tests { frames: vec![FrameInfo { jitcode_index: 0, pc: 100, + py_pc: -1, slot_map: vec![ FrameSlotSource::FailArg(2), FrameSlotSource::Unavailable, @@ -4604,11 +4620,13 @@ mod tests { FrameInfo { jitcode_index: 0, pc: 10, + py_pc: -1, slot_map: vec![FrameSlotSource::FailArg(0), FrameSlotSource::FailArg(1)], }, FrameInfo { jitcode_index: 1, pc: 20, + py_pc: -1, slot_map: vec![FrameSlotSource::FailArg(2), FrameSlotSource::FailArg(3)], }, ], @@ -4635,7 +4653,7 @@ mod tests { #[test] fn test_builder() { let mut builder = ResumeDataVirtualAdder::new(); - builder.push_frame(0, 42); + builder.push_frame(0, 42, -1); builder.map_slot(0, 0); builder.map_slot(2, 1); // gap at slot 1 let rd = builder.build(); @@ -4921,11 +4939,13 @@ mod tests { SnapshotFrame { jitcode_index: 0, pc: 10, + py_pc: 10, boxes: vec![OpRef::int_op(1).into(), OpRef::const_int(99).into()], }, SnapshotFrame { jitcode_index: 1, pc: 20, + py_pc: 20, boxes: vec![OpRef::int_op(2).into(), OpRef::int_op(3).into()], }, ], @@ -4938,12 +4958,16 @@ mod tests { // Multi-frame encoding: no box_count, RPython parity. let items = crate::resumecode::unpack_numbering(&rd_numb); assert_eq!(items[1], 3); // num_failargs: 3 boxes patched - // Frame 0: items[4]=jitcode(0), items[5]=pc(10), items[6..7]=tagged + // Frame 0: items[4]=jitcode(0), items[5]=pc(10), items[6]=py_pc(10), + // items[7..8]=tagged. assert_eq!(items[4], 0); assert_eq!(items[5], 10); - // Frame 1: items[8]=jitcode(1), items[9]=pc(20), items[10..11]=tagged - assert_eq!(items[8], 1); - assert_eq!(items[9], 20); + assert_eq!(items[6], 10); + // Frame 1: items[9]=jitcode(1), items[10]=pc(20), items[11]=py_pc(20), + // items[12..13]=tagged. + assert_eq!(items[9], 1); + assert_eq!(items[10], 20); + assert_eq!(items[11], 20); // Roundtrip with liveness-based closure. let rd_consts: Vec = memo.consts().to_vec(); @@ -4965,9 +4989,11 @@ mod tests { assert_eq!(rebuilt_frames.len(), 2); assert_eq!(rebuilt_frames[0].jitcode_index, 0); assert_eq!(rebuilt_frames[0].pc, 10); + assert_eq!(rebuilt_frames[0].py_pc, 10); assert_eq!(rebuilt_frames[0].values.len(), 2); assert_eq!(rebuilt_frames[1].jitcode_index, 1); assert_eq!(rebuilt_frames[1].pc, 20); + assert_eq!(rebuilt_frames[1].py_pc, 20); assert_eq!(rebuilt_frames[1].values.len(), 2); } @@ -5036,6 +5062,7 @@ mod tests { framestack: vec![SnapshotFrame { jitcode_index: 0, pc: 8, + py_pc: 8, boxes: vec![OpRef::int_op(1).into()], }], }; @@ -6366,10 +6393,12 @@ impl<'a> ResumeDataDirectReader<'a> { // ---- AbstractResumeDataReader methods (resume.py:928-1038) ---- - /// resume.py:928 read_jitcode_pos_pc. Returns `(jitcode_pos, pc)`. + /// resume.py:928 read_jitcode_pos_pc. The wire header also carries + /// forward `py_pc`, which blackhole does not use but must consume. pub fn read_jitcode_pos_pc(&mut self) -> (i32, i32) { let jitcode_pos = self.resumecodereader.next_item(); let pc = self.resumecodereader.next_item(); + let _py_pc = self.resumecodereader.next_item(); (jitcode_pos, pc) } diff --git a/majit/majit-metainterp/tests/jit_driver_runtime_parity.rs b/majit/majit-metainterp/tests/jit_driver_runtime_parity.rs index 87003b78024..c22d1a9ce86 100644 --- a/majit/majit-metainterp/tests/jit_driver_runtime_parity.rs +++ b/majit/majit-metainterp/tests/jit_driver_runtime_parity.rs @@ -27,6 +27,7 @@ fn attach_single_frame_snapshot(ctx: &mut TraceCtx, pc: u32, boxes: &[(OpRef, Ty frames: vec![SnapshotFrame { jitcode_index: 0, pc, + py_pc: pc, boxes: boxes .iter() .map(|(opref, tp)| SnapshotTagged::Box(*opref, *tp)) @@ -1611,7 +1612,7 @@ fn jit_state_restore_guard_failure_restores_from_reconstructed_resume_frame() { }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 444); + resume.push_frame(0, 444, -1); resume.set_slot_constant(0, majit_ir::Const::Ref(GcRef(frame_ptr as usize))); resume.map_slot(1, 0); resume.set_slot_constant(2, majit_ir::Const::Int(99)); @@ -1645,7 +1646,7 @@ fn jit_state_restore_guard_failure_materializes_virtual_ref_from_resume_state() }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 555); + resume.push_frame(0, 555, -1); let virtual_index = resume.add_virtual_struct( Some(typedescr_7), 0, @@ -1692,7 +1693,7 @@ fn jit_state_restore_guard_failure_materializes_nested_virtual_refs_in_dependenc }; let meta = TestMeta { header_pc: 556 }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 556); + resume.push_frame(0, 556, -1); let inner = resume.add_virtual_struct( Some(inner_typedescr), 0, @@ -1761,7 +1762,7 @@ fn jit_state_restore_guard_failure_replays_pending_writes_with_virtual_target_an }; let meta = TestMeta { header_pc: 557 }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 557); + resume.push_frame(0, 557, -1); let parent = resume.add_virtual_struct(Some(parent_typedescr), 0, vec![], vec![], 0); let child = resume.add_virtual_struct(Some(child_typedescr), 0, vec![], vec![], 0); resume.set_slot_virtual(0, parent); @@ -1799,7 +1800,7 @@ fn jit_state_restore_guard_failure_replays_pending_field_writes() { let mut state = PendingWriteState { obj: 0, flag: 1 }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 666); + resume.push_frame(0, 666, -1); resume.set_slot_constant(0, majit_ir::Const::Ref(GcRef(cell_ptr))); resume.map_slot(1, 0); let pending_descr: majit_ir::DescrRef = std::sync::Arc::new( @@ -1835,7 +1836,7 @@ fn jit_state_restore_guard_failure_replays_pending_array_writes_via_layout_hook( let mut state = PendingArrayWriteState { array: 0, flag: 1 }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 888); + resume.push_frame(0, 888, -1); resume.set_slot_constant(0, majit_ir::Const::Ref(GcRef(array_ptr))); resume.map_slot(1, 0); let pending_descr: majit_ir::DescrRef = std::sync::Arc::new( @@ -1875,10 +1876,10 @@ fn jit_state_restore_guard_failure_can_restore_multi_frame_resume_state() { }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 100); + resume.push_frame(0, 100, -1); resume.set_slot_constant(0, majit_ir::Const::Ref(GcRef(frame_ptr as usize))); resume.set_slot_constant(1, majit_ir::Const::Int(1)); - resume.push_frame(0, 200); + resume.push_frame(0, 200, -1); resume.set_slot_constant(0, majit_ir::Const::Ref(GcRef(frame_ptr as usize))); resume.map_slot(1, 0); let reconstructed_state = resume.build().reconstruct_state(&[2]); @@ -1911,11 +1912,11 @@ fn jit_state_restore_guard_failure_can_restore_multi_frame_state_via_generic_fra }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 300); + resume.push_frame(0, 300, -1); let virtual_index = resume.add_virtual_struct(None, 0, vec![], vec![], 0); resume.set_slot_virtual(0, virtual_index); resume.set_slot_constant(1, majit_ir::Const::Int(1)); - resume.push_frame(0, 400); + resume.push_frame(0, 400, -1); resume.set_slot_virtual(0, virtual_index); resume.map_slot(1, 0); let reconstructed_state = resume.build().reconstruct_state(&[2]); @@ -1954,11 +1955,11 @@ fn jit_state_restore_guard_failure_reuses_virtual_cache_for_pending_writes() { }; let mut resume = ResumeDataVirtualAdder::new(); - resume.push_frame(0, 500); + resume.push_frame(0, 500, -1); let virtual_index = resume.add_virtual_struct(None, 0, vec![], vec![], 0); resume.set_slot_virtual(0, virtual_index); resume.set_slot_constant(1, majit_ir::Const::Int(1)); - resume.push_frame(0, 600); + resume.push_frame(0, 600, -1); resume.set_slot_virtual(0, virtual_index); resume.map_slot(1, 0); let pending_descr: majit_ir::DescrRef = std::sync::Arc::new( diff --git a/majit/majit-metainterp/tests/resume_parity.rs b/majit/majit-metainterp/tests/resume_parity.rs index 0f14f7fac65..de85730e050 100644 --- a/majit/majit-metainterp/tests/resume_parity.rs +++ b/majit/majit-metainterp/tests/resume_parity.rs @@ -24,6 +24,7 @@ fn resume_py_public_encoding_uses_tagged_numbering() { frames: vec![FrameInfo { jitcode_index: 0, pc: 123, + py_pc: 321, slot_map: vec![ FrameSlotSource::FailArg(0), FrameSlotSource::Constant(Const::Int(7)), @@ -45,16 +46,18 @@ fn resume_py_public_encoding_uses_tagged_numbering() { assert_eq!(encoded.rd_numb[0] as usize, encoded.rd_numb.len()); assert_eq!(encoded.rd_consts, vec![Const::Int(large_const)]); // Header layout (resume.py:231-253): - // [size, count, vable_array_len, vref_array_len, jitcode_index, pc, ...slots] + // [size, count, vable_array_len, vref_array_len, jitcode_index, pc, py_pc, ...slots] // rd_numb[1] = count: 1 livebox (FailArg(0) in frame slot) assert_eq!(encoded.rd_numb[1], 1); - let slot_words = &encoded.rd_numb[6..12]; + assert_eq!(encoded.rd_numb[6], 321); + let slot_words = &encoded.rd_numb[7..13]; assert_eq!(untag(slot_words[0]), (0, TAG_BOX)); assert_eq!(untag(slot_words[1]), (7, TAG_INT)); assert_eq!(untag(slot_words[2]), (0, TAG_CONST)); assert_eq!(untag(slot_words[3]), (0, TAG_VIRTUAL)); assert_eq!(untag(slot_words[4]), (ENCODED_UNINITIALIZED, TAG_CONST)); assert_eq!(untag(slot_words[5]), (ENCODED_UNAVAILABLE, TAG_CONST)); + assert_eq!(encoded.decode().frames[0].py_pc, 321); } #[test] @@ -67,6 +70,7 @@ fn resume_py_public_roundtrip_recovers_virtualized_state() { frames: vec![FrameInfo { jitcode_index: 0, pc: 77, + py_pc: -1, slot_map: vec![ FrameSlotSource::FailArg(0), FrameSlotSource::Constant(Const::Int(42)), @@ -152,6 +156,7 @@ fn resume_py_count_includes_virtual_and_pending_field_failargs() { frames: vec![FrameInfo { jitcode_index: 0, pc: 10, + py_pc: -1, slot_map: vec![ FrameSlotSource::FailArg(0), FrameSlotSource::Constant(Const::Int(42)), @@ -187,6 +192,7 @@ fn resume_py_count_frame_only() { frames: vec![FrameInfo { jitcode_index: 0, pc: 10, + py_pc: -1, slot_map: vec![ FrameSlotSource::FailArg(0), FrameSlotSource::FailArg(1), @@ -211,6 +217,7 @@ fn resume_py_compact_liveboxes_numbering() { frames: vec![FrameInfo { jitcode_index: 0, pc: 10, + py_pc: -1, slot_map: vec![ FrameSlotSource::FailArg(0), FrameSlotSource::FailArg(7), @@ -226,8 +233,8 @@ fn resume_py_compact_liveboxes_numbering() { // liveboxes[0] = 0, liveboxes[1] = 7 assert_eq!(encoded.liveboxes, vec![0, 7]); // TAGBOX(0) for FailArg(0), TAGBOX(1) for FailArg(7) - // Slots start after the two-word frame header. - let slot_words = &encoded.rd_numb[6..9]; + // Slots start after the three-word frame header. + let slot_words = &encoded.rd_numb[7..10]; assert_eq!(untag(slot_words[0]), (0, TAG_BOX)); assert_eq!(untag(slot_words[1]), (1, TAG_BOX)); assert_eq!(untag(slot_words[2]), (42, TAG_INT)); @@ -247,6 +254,7 @@ fn resume_py_dedup_same_box_same_number() { frames: vec![FrameInfo { jitcode_index: 0, pc: 10, + py_pc: -1, slot_map: vec![ FrameSlotSource::FailArg(5), FrameSlotSource::FailArg(5), @@ -261,8 +269,8 @@ fn resume_py_dedup_same_box_same_number() { assert_eq!(encoded.rd_numb[1], 2); assert_eq!(encoded.liveboxes, vec![5, 3]); // Both FailArg(5) slots get TAGBOX(0), FailArg(3) gets TAGBOX(1) - // Slots start after the two-word frame header. - let slot_words = &encoded.rd_numb[6..9]; + // Slots start after the three-word frame header. + let slot_words = &encoded.rd_numb[7..10]; assert_eq!(untag(slot_words[0]), (0, TAG_BOX)); assert_eq!(untag(slot_words[1]), (0, TAG_BOX)); assert_eq!(untag(slot_words[2]), (1, TAG_BOX)); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs index ee84a11f9d0..3f14ee7df4a 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs @@ -11,6 +11,14 @@ use super::*; +/// `PYRE_M73_PYPC_FWD_AUDIT`: verify the forward Python-PC resume word +/// against the still-live decode-side back-translation. Cached once, like +/// the other walker diagnostics; the audit emits no per-event logging. +pub fn py_pc_forward_audit_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var_os("PYRE_M73_PYPC_FWD_AUDIT").is_some()) +} + /// `PYRE_PCMAP_RECIPE_RESULTCOLOR_AUDIT` is a report-only census for the /// recipe resume-coordinate result-color reader and the multi-frame callee /// diagnostic's inversion. The optional `_PROBE` receives a fire row followed diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index d6d9cae664d..6b00ca26a92 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -11,6 +11,19 @@ use super::*; +/// Read the Python PC paired with a native resume word. The codewriter builds +/// this table from Python-PC keys, including the same forward trivia skip as +/// `backxlat_py_pc`; capture deliberately never projects the word backwards. +fn forward_snapshot_py_pc(jitcode_index: u32, pc: u32) -> Result { + if pc == majit_ir::resumedata::NO_JITCODE_PC as u32 { + return Ok(u32::MAX); + } + crate::state::pyjitcode_for_jitcode_index(jitcode_index as i32) + .and_then(|payload| payload.resume_position_for_jitcode_pc(pc as usize)) + .map(|(_, py_pc)| py_pc) + .ok_or(DispatchError::GuardResumeCoordinateUnavailable { pc: pc as usize }) +} + /// `generate_guard` (`pyjitpl.py`) keys `after_residual_call` /// on the guard opcode itself: `GUARD_EXCEPTION` / `GUARD_NO_EXCEPTION` / /// `GUARD_NOT_FORCED` / `GUARD_ALWAYS_FAILS` resume *after* the residual @@ -169,11 +182,13 @@ pub(crate) fn walker_capture_inline_nonstandard_vable_guard( return Err(DispatchError::GuardResumeCoordinateUnavailable { pc: op_pc }); }; let (vable_boxes, vref_boxes) = ctx.trace_ctx.build_snapshot_vable_vref_boxes(); + let nsvable_py_pc = forward_snapshot_py_pc(ctx.outer_jitcode_index, nsvable_pc_word)?; ctx.trace_ctx .capture_snapshot_for_last_guard_op_with_vable_vref( &ctx.outer_active_boxes, ctx.outer_jitcode_index, nsvable_pc_word, + nsvable_py_pc, &vable_boxes, &vref_boxes, ); @@ -972,11 +987,13 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( scope.branch_guard_kept_recovered, ); let pc_word = resolved_offset as u32; + let forward_py_pc = forward_snapshot_py_pc(jitcode_index, pc_word)?; ctx.trace_ctx .capture_snapshot_for_last_guard_with_vable_vref( &active, jitcode_index, pc_word, + forward_py_pc, &vable_boxes, &vref_boxes, ); @@ -1003,11 +1020,13 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( else { return Err(DispatchError::GuardResumeCoordinateUnavailable { pc: op_pc }); }; + let arm_py_pc = forward_snapshot_py_pc(ctx.outer_jitcode_index, arm_pc_word)?; ctx.trace_ctx .capture_snapshot_for_last_guard_with_vable_vref( &ctx.outer_active_boxes, ctx.outer_jitcode_index, arm_pc_word, + arm_py_pc, &vable_boxes, &vref_boxes, ); @@ -1764,7 +1783,8 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( // Frame tuples, OUTERMOST-FIRST: the paused caller chain, then the callee // top frame last (innermost). - let mut frames: Vec<(u32, u32, &[OpRef])> = Vec::with_capacity(parent_frames.len() + 1); + let mut frames: Vec<(u32, u32, u32, &[OpRef])> = + Vec::with_capacity(parent_frames.len() + 1); for pf in &parent_frames { let pf_word = pf .resume_marker_jit_pc @@ -1780,11 +1800,15 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( else { return Err(DispatchError::GuardResumeCoordinateUnavailable { pc: callee_op_pc }); }; - frames.push((pf.jitcode_index, pf_pc_word, pf.boxes.as_slice())); + let pf_py_pc = forward_snapshot_py_pc(pf.jitcode_index, pf_pc_word)?; + frames.push((pf.jitcode_index, pf_pc_word, pf_py_pc, pf.boxes.as_slice())); } + let callee_py_pc = + forward_snapshot_py_pc(callee_jitcode_index as u32, callee_jitcode_pc as u32)?; frames.push(( callee_jitcode_index as u32, callee_jitcode_pc as u32, + callee_py_pc, callee_boxes.as_slice(), )); diff --git a/pyre/pyre-jit-trace/src/pyjitcode.rs b/pyre/pyre-jit-trace/src/pyjitcode.rs index a6918a652a9..c9ad2f4d24d 100644 --- a/pyre/pyre-jit-trace/src/pyjitcode.rs +++ b/pyre/pyre-jit-trace/src/pyjitcode.rs @@ -58,6 +58,13 @@ use std::ops::{Deref, DerefMut}; /// translation maps live here instead of polluting either upstream's /// canonical `JitCode` or pyre's eventual single-store replacement. pub struct PyJitCodeMetadata { + /// Codewrite-time forward Python-PC twin for resume data. The exact + /// block-head and predecessor-op-start tiers mirror + /// `python_pc_for_jitcode_pc` and already include the forward trivia + /// normalization. It is populated from Python keys while codewriting; + /// snapshot capture must not invert a resolved JitCode PC to obtain it. + pub forward_py_pc_marker_by_jit_pc: Vec<(usize, u32)>, + pub forward_py_pc_pred_by_jit_pc: Vec<(usize, u32)>, /// Post-residual-call catch resume twin, split into the exact /// block-head-marker and predecessor op-start tiers used by /// `python_pc_for_jitcode_pc`. Empty for skeleton / fixture metadata. @@ -710,6 +717,30 @@ impl PyJitCode { Self::predecessor_index(search).and_then(|i| pred[i].1) } + /// Forward Python PC paired with a carried resume JitCode PC. This is a + /// codewriter-built twin of `backxlat_py_pc`'s result, including its + /// forward trivia normalization, not an encode-time inverse projection. + pub fn forward_py_pc_for_jitcode_pc(&self, jit_pc: usize) -> Option { + let marker = &self.metadata.forward_py_pc_marker_by_jit_pc; + let pred = &self.metadata.forward_py_pc_pred_by_jit_pc; + if marker.is_empty() && pred.is_empty() { + return None; + } + if let Ok(i) = marker.binary_search_by_key(&jit_pc, |&(off, _)| off) { + return Some(marker[i].1); + } + let search = pred.binary_search_by_key(&jit_pc, |&(off, _)| off); + Self::predecessor_index(search).map(|i| pred[i].1) + } + + /// Return the codewriter-carried `(jitcode_pc, py_pc)` resume pair for a + /// native resume coordinate. The first member remains the primary + /// liveness/setposition key; the second is solely forward resume data. + pub fn resume_position_for_jitcode_pc(&self, jit_pc: usize) -> Option<(usize, u32)> { + self.forward_py_pc_for_jitcode_pc(jit_pc) + .map(|py_pc| (jit_pc, py_pc)) + } + /// Codewrite-time after-residual fallthrough marker /// keyed by a JitCode byte offset, resolved with the SAME two tiers as /// `python_pc_for_jitcode_pc`: an EXACT marker match first (block-head @@ -891,6 +922,8 @@ impl PyJitCode { Self::from_parts( std::sync::Arc::new(RuntimeJitCode::default()), PyJitCodeMetadata { + forward_py_pc_marker_by_jit_pc: Vec::new(), + forward_py_pc_pred_by_jit_pc: Vec::new(), after_residual_call_resume_marker_by_jit_pc: Vec::new(), after_residual_call_resume_pred_by_jit_pc: Vec::new(), n_py_instrs: 0, diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 96c2d921bb8..2d816cf00d9 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -11620,6 +11620,8 @@ mod tests { let pyjit = std::sync::Arc::new(crate::PyJitCode::from_parts( runtime_jitcode, crate::PyJitCodeMetadata { + forward_py_pc_marker_by_jit_pc: Vec::new(), + forward_py_pc_pred_by_jit_pc: Vec::new(), after_residual_call_resume_marker_by_jit_pc: Vec::new(), after_residual_call_resume_pred_by_jit_pc: Vec::new(), n_py_instrs: 0, @@ -11723,6 +11725,7 @@ mod tests { frames: vec![RebuiltFrame { jitcode_index, pc: 0, + py_pc: 0, values: vec![ RebuiltValue::Box(8, Type::Ref), RebuiltValue::Box(9, Type::Ref), diff --git a/pyre/pyre-jit-trace/src/trace_opcode.rs b/pyre/pyre-jit-trace/src/trace_opcode.rs index a7e0ba6b2d7..06344ffd49f 100644 --- a/pyre/pyre-jit-trace/src/trace_opcode.rs +++ b/pyre/pyre-jit-trace/src/trace_opcode.rs @@ -2913,9 +2913,14 @@ impl MIFrame { crate::state::request_trace_abort(); top_pc as u32 }); + let top_py_pc = resolved + .and_then(|offset| payload.resume_position_for_jitcode_pc(offset)) + .map(|(_, py_pc)| py_pc) + .unwrap_or(top_pc as u32); let top_frame = majit_metainterp::recorder::SnapshotFrame { jitcode_index: top_jitcode_index, pc: top_pc_word, + py_pc: top_py_pc, boxes: Self::fail_args_to_snapshot_boxes_typed( top_active_boxes, top_snapshot_types, diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index b186e8e9e98..888a62a04f5 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -1029,7 +1029,6 @@ pub struct ResumedFrame { /// Some(pc): snapshot guard — orgpc known, liveness-based filling. /// pc=0 is valid (function start / loop header at bytecode 0). /// None: no-snapshot guard (rd_numb pc=-1), positional fallback. - pub rd_numb_pc: Option, /// CHAIN virtualizable pointer (same value on every section). /// RPython parity: there is ONE virtualizable per jitdriver_sd for the /// whole blackhole chain; inner sections do not own a separate PyFrame. diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index c23556a7de4..3e36a7c91ca 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -9435,6 +9435,16 @@ fn build_resumed_frames( // pc=0 is valid (function start). pc=-1 = no-snapshot sentinel. let decoded_py_pc = (frame.pc >= 0) .then(|| pyre_jit_trace::state::backxlat_py_pc(frame.jitcode_index, frame.pc) as usize); + if pyre_jit_trace::jitcode_dispatch::py_pc_forward_audit_enabled() { + let expected_py_pc = decoded_py_pc.map(|pc| pc as i32).unwrap_or(-1); + assert_eq!( + frame.py_pc, + expected_py_pc, + "PYRE_M73_PYPC_FWD_AUDIT: forward py_pc diverged for jitcode {} jitcode_pc {}", + frame.jitcode_index, + frame.pc, + ); + } let py_pc = decoded_py_pc.unwrap_or(vable_ni); // resume.py:1339 jitcodes[jitcode_pos]: // Outermost frame: code from vable resume data. @@ -9508,7 +9518,6 @@ fn build_resumed_frames( result.push(crate::call_jit::ResumedFrame { code: w_code, py_pc, - rd_numb_pc: decoded_py_pc, frame_ptr, vsd, namespace, diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index f82c829f0f6..3ce0fbd012e 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -13652,6 +13652,8 @@ impl CodeWriter { let mut result_color_after_residual_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut depth_after_residual_marker_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut depth_after_residual_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); + let mut forward_py_pc_marker_by_jit_pc: Vec<(usize, u32)> = Vec::new(); + let mut forward_py_pc_pred_by_jit_pc: Vec<(usize, u32)> = Vec::new(); let mut after_residual_call_resume_marker_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut after_residual_call_resume_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); @@ -13686,6 +13688,9 @@ impl CodeWriter { // corrected block-entry PC from the inversion table so the direct // depth twin and `python_pc_for_jitcode_pc` cannot diverge. for &(off, py) in &block_head_py_by_jit_pc { + let skipped_py = + pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py as usize); + forward_py_pc_marker_by_jit_pc.push((off, skipped_py as u32)); let skipped_py = pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py as usize); let depth_trivia = static_depth.get(skipped_py).copied(); @@ -13708,6 +13713,7 @@ impl CodeWriter { result_color_trivia_marker_by_jit_pc.push((off, result_color_trivia)); } depth_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + forward_py_pc_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); pcdep_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); const_ref_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); @@ -13716,6 +13722,7 @@ impl CodeWriter { if pos != usize::MAX { let skipped_py = pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py); + forward_py_pc_pred_by_jit_pc.push((pos, skipped_py as u32)); let depth_trivia = static_depth.get(skipped_py).copied(); depth_trivia_pred_by_jit_pc.push((pos, depth_trivia)); pcdep_trivia_pred_by_jit_pc.push(( @@ -13737,6 +13744,7 @@ impl CodeWriter { } } depth_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + forward_py_pc_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); pcdep_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); const_ref_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); @@ -13849,6 +13857,8 @@ impl CodeWriter { let frame_stack_base = code.varnames.len() + pyre_interpreter::pyframe::ncells(code); let metadata = PyJitCodeMetadata { + forward_py_pc_marker_by_jit_pc, + forward_py_pc_pred_by_jit_pc, after_residual_call_resume_marker_by_jit_pc, after_residual_call_resume_pred_by_jit_pc, n_py_instrs, From 13138a78efd2026507c090cf025fe4c1c6e201a8 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Jul 2026 21:42:42 +0900 Subject: [PATCH 2/8] jit: cut guard-fail resume to the forward py_pc, drop the decode-side backxlat (jitcode-blackhole Slice 3' Phase B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `build_resumed_frames` now reads each frame's forward-carried `py_pc` from the resume data instead of deriving it via `backxlat_py_pc(jitcode_index, pc)` at decode. The Phase-A `PYRE_M73_PYPC_FWD_AUDIT` proved the two equal across the corpus (248/248, zero divergence), so the cutover preserves behavior. This removes the last resume-decode jitcode→python-pc reentry inverse: the `backxlat_py_pc` call site in `build_resumed_frames` is gone (the function keeps its bridge-trace-side callers). Drops the now-obsolete audit helper. check.py: dynasm 248/248 + cranelift 248/248. Assisted-by: Claude --- majit/majit-metainterp/src/history.rs | 5 +++- .../src/jitcode_dispatch/diag.rs | 8 ------- .../src/jitcode_dispatch/resume_snapshot.rs | 3 +-- pyre/pyre-jit/src/eval.rs | 23 ++++++++----------- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/majit/majit-metainterp/src/history.rs b/majit/majit-metainterp/src/history.rs index e00acbd6569..f5820550d0a 100644 --- a/majit/majit-metainterp/src/history.rs +++ b/majit/majit-metainterp/src/history.rs @@ -2320,7 +2320,10 @@ impl TraceCtx { /// (RPython's `_number_boxes` does this implicitly via the memo /// table; pyre's `Snapshot.encode` does the same in /// `resume.rs:1898 _number_boxes`). - pub fn capture_snapshot_for_last_guard_multi_frame(&mut self, frames: &[(u32, u32, u32, &[OpRef])]) { + pub fn capture_snapshot_for_last_guard_multi_frame( + &mut self, + frames: &[(u32, u32, u32, &[OpRef])], + ) { self.capture_snapshot_for_last_guard_multi_frame_with_vable_vref(frames, &[], &[]); } diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs index 3f14ee7df4a..ee84a11f9d0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs @@ -11,14 +11,6 @@ use super::*; -/// `PYRE_M73_PYPC_FWD_AUDIT`: verify the forward Python-PC resume word -/// against the still-live decode-side back-translation. Cached once, like -/// the other walker diagnostics; the audit emits no per-event logging. -pub fn py_pc_forward_audit_enabled() -> bool { - static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); - *ENABLED.get_or_init(|| std::env::var_os("PYRE_M73_PYPC_FWD_AUDIT").is_some()) -} - /// `PYRE_PCMAP_RECIPE_RESULTCOLOR_AUDIT` is a report-only census for the /// recipe resume-coordinate result-color reader and the multi-frame callee /// diagnostic's inversion. The optional `_PROBE` receives a fire row followed diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index 6b00ca26a92..fca08600265 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -1783,8 +1783,7 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( // Frame tuples, OUTERMOST-FIRST: the paused caller chain, then the callee // top frame last (innermost). - let mut frames: Vec<(u32, u32, u32, &[OpRef])> = - Vec::with_capacity(parent_frames.len() + 1); + let mut frames: Vec<(u32, u32, u32, &[OpRef])> = Vec::with_capacity(parent_frames.len() + 1); for pf in &parent_frames { let pf_word = pf .resume_marker_jit_pc diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 3e36a7c91ca..e97186fd8d8 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -9432,20 +9432,15 @@ fn build_resumed_frames( let mut result = Vec::with_capacity(frames.len()); for (idx, (frame, values)) in frames.iter().zip(all_values.into_iter()).enumerate() { - // pc=0 is valid (function start). pc=-1 = no-snapshot sentinel. - let decoded_py_pc = (frame.pc >= 0) - .then(|| pyre_jit_trace::state::backxlat_py_pc(frame.jitcode_index, frame.pc) as usize); - if pyre_jit_trace::jitcode_dispatch::py_pc_forward_audit_enabled() { - let expected_py_pc = decoded_py_pc.map(|pc| pc as i32).unwrap_or(-1); - assert_eq!( - frame.py_pc, - expected_py_pc, - "PYRE_M73_PYPC_FWD_AUDIT: forward py_pc diverged for jitcode {} jitcode_pc {}", - frame.jitcode_index, - frame.pc, - ); - } - let py_pc = decoded_py_pc.unwrap_or(vable_ni); + // Forward-carried Python resume pc, recorded at guard capture from the + // codewriter `(jitcode_pc, py_pc)` marker pair (no jitcode→py inverse at + // decode). py_pc=-1 is the no-snapshot sentinel (pc<0) → fall back to + // the vable next-instr. + let py_pc = if frame.py_pc >= 0 { + frame.py_pc as usize + } else { + vable_ni + }; // resume.py:1339 jitcodes[jitcode_pos]: // Outermost frame: code from vable resume data. // Inner frames: code from jitcode_index registry (inlined calls). From c72b3c9c07b50a8cd7d9e344f819051154f85807 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2026 01:25:17 +0900 Subject: [PATCH 3/8] jit: fix resume-data unit tests for the forward py_pc frame header Follow-on to the Slice 3' wire-format change (`jitcode_index, pc, py_pc` frame header): repair the tests that were still shaped for the two-word `(jitcode_index, pc)` header. - pyjitpl.rs: add the missing `py_pc` field to the `recorder::SnapshotFrame` built in `finish_trace_for_parity_preserves_captured_snapshots` (this was a `cargo test` compile break; the lib build did not cover it). - resume.rs: shift the hardcoded wire-offset assertions in five numbering tests by the inserted per-frame `py_pc` word and assert its value. - jitcode_dispatch/tests.rs: populate `forward_py_pc_marker_by_jit_pc` / `forward_py_pc_pred_by_jit_pc` on the guard-resume test JitCode so guard capture resolves the innermost frame's forward Python pc instead of aborting with `GuardResumeCoordinateUnavailable`. - history.rs: document that `capture_snapshot_for_last_guard` sets `py_pc == pc` because its native-driver / test callers have no CPython-bytecode coordinate split. check.py stays 248/248; majit-metainterp and pyre-jit test suites pass. Assisted-by: Claude --- majit/majit-metainterp/src/history.rs | 7 ++++ majit/majit-metainterp/src/pyjitpl.rs | 1 + majit/majit-metainterp/src/resume.rs | 40 +++++++++++-------- .../src/jitcode_dispatch/tests.rs | 6 +++ 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/majit/majit-metainterp/src/history.rs b/majit/majit-metainterp/src/history.rs index f5820550d0a..23cd03eaebb 100644 --- a/majit/majit-metainterp/src/history.rs +++ b/majit/majit-metainterp/src/history.rs @@ -2220,6 +2220,13 @@ impl TraceCtx { /// resolve to a known `Box.type`; constants must have a recorded /// value. Misses are bookkeeping bugs and panic, not silent /// fallbacks. + /// `py_pc == pc` here: this convenience serves jitdrivers whose + /// interpreter pc already *is* the JitCode pc — the native meta-tracing + /// clients (and unit tests) that have no CPython-bytecode layer, so the + /// two coordinates coincide and no JitCode→Python translation applies. + /// The pyre CPython-bytecode path never uses this shortcut; it carries a + /// distinct forward Python pc through + /// `capture_snapshot_for_last_guard_with_vable_vref`. pub fn capture_snapshot_for_last_guard( &mut self, active_boxes: &[OpRef], diff --git a/majit/majit-metainterp/src/pyjitpl.rs b/majit/majit-metainterp/src/pyjitpl.rs index 23748437256..34632e3f0d7 100644 --- a/majit/majit-metainterp/src/pyjitpl.rs +++ b/majit/majit-metainterp/src/pyjitpl.rs @@ -20465,6 +20465,7 @@ mod tests { frames: vec![crate::recorder::SnapshotFrame { jitcode_index: 0, pc: 123, + py_pc: 123, boxes: vec![crate::recorder::SnapshotTagged::Box( OpRef::int_op(0), majit_ir::Type::Int, diff --git a/majit/majit-metainterp/src/resume.rs b/majit/majit-metainterp/src/resume.rs index 2ab3672c2a6..0bbb79b8f2a 100644 --- a/majit/majit-metainterp/src/resume.rs +++ b/majit/majit-metainterp/src/resume.rs @@ -4681,7 +4681,7 @@ mod tests { vec![OpRef::const_int(42), OpRef::int_op(1), OpRef::int_op(2)], ); let numb_state = memo.number(&snapshot, &env, -1).unwrap(); - // Should have: [size, num_failargs, 0(vable), 0(vref), 0(jitcode), 8(pc), tagged...] + // Should have: [size, num_failargs, 0(vable), 0(vref), 0(jitcode), 8(pc), 8(py_pc), tagged...] let items = crate::resumecode::unpack_numbering(&numb_state.create_numbering()); // items[0] = total size assert!(items[0] > 0); @@ -4696,16 +4696,18 @@ mod tests { assert_eq!(items[4], 0); // items[5] = pc = 8 assert_eq!(items[5], 8); - // items[6] = inline-Const(42) tagged as TAGINT(42) since 42 fits in 13 bits - let (val, tagbits) = untag(items[6] as i16); + // items[6] = py_pc = 8 (forward-carried Python pc; single_frame sets py_pc = pc) + assert_eq!(items[6], 8); + // items[7] = inline-Const(42) tagged as TAGINT(42) since 42 fits in 13 bits + let (val, tagbits) = untag(items[7] as i16); assert_eq!(tagbits, TAGINT); assert_eq!(val, 42); - // items[7] = OpRef::int_op(1) tagged as TAGBOX(0) — first live box - let (val, tagbits) = untag(items[7] as i16); + // items[8] = OpRef::int_op(1) tagged as TAGBOX(0) — first live box + let (val, tagbits) = untag(items[8] as i16); assert_eq!(tagbits, TAGBOX); assert_eq!(val, 0); - // items[8] = OpRef::int_op(2) tagged as TAGBOX(1) — second live box - let (val, tagbits) = untag(items[8] as i16); + // items[9] = OpRef::int_op(2) tagged as TAGBOX(1) — second live box + let (val, tagbits) = untag(items[9] as i16); assert_eq!(tagbits, TAGBOX); assert_eq!(val, 1); } @@ -4796,16 +4798,18 @@ mod tests { let items = crate::resumecode::unpack_numbering(&numb_state.create_numbering()); // items[1] = num_failargs: 0 (not patched — RPython patches in finish()) assert_eq!(items[1], 0); - // items[6] = OpRef::int_op(1) → TAGBOX(0) - let (val, tagbits) = untag(items[6] as i16); + // items[6] = py_pc = 10 (single_frame sets py_pc = pc) + assert_eq!(items[6], 10); + // items[7] = OpRef::int_op(1) → TAGBOX(0) + let (val, tagbits) = untag(items[7] as i16); assert_eq!(tagbits, TAGBOX); assert_eq!(val, 0); - // items[7] = OpRef::ref_op(2) → TAGVIRTUAL(0) - let (val, tagbits) = untag(items[7] as i16); + // items[8] = OpRef::ref_op(2) → TAGVIRTUAL(0) + let (val, tagbits) = untag(items[8] as i16); assert_eq!(tagbits, TAGVIRTUAL); assert_eq!(val, 0); - // items[8] = OpRef::int_op(3) → TAGBOX(1) - let (val, tagbits) = untag(items[8] as i16); + // items[9] = OpRef::int_op(3) → TAGBOX(1) + let (val, tagbits) = untag(items[9] as i16); assert_eq!(tagbits, TAGBOX); assert_eq!(val, 1); } @@ -4918,8 +4922,10 @@ mod tests { let numb_state = memo.number(&snapshot, &env, -1).unwrap(); let items = crate::resumecode::unpack_numbering(&numb_state.create_numbering()); - // items[6] = the frame's box. - let (val, tagbits) = untag(items[6] as i16); + // items[6] = py_pc = 10 (single_frame sets py_pc = pc) + assert_eq!(items[6], 10); + // items[7] = the frame's box. + let (val, tagbits) = untag(items[7] as i16); assert_eq!(tagbits, TAGVIRTUAL); assert_eq!(val, 0); assert_eq!(numb_state.num_boxes, 0); @@ -5082,11 +5088,12 @@ mod tests { assert_eq!(items[5], 0); // vref_array_length assert_eq!(items[6], 0); // jitcode_index assert_eq!(items[7], 8); // pc + assert_eq!(items[8], 8); // py_pc (single_frame sets py_pc = pc) // The frame slot reuses the payload tag because numbering follows // Box identity exactly: upstream dedups only when the same Box object // appears twice, and in this test we passed the same OpRef twice. - let (val, tagbits) = untag(items[8] as i16); + let (val, tagbits) = untag(items[9] as i16); assert_eq!(tagbits, TAGBOX); assert_eq!(val, 0); } @@ -5104,6 +5111,7 @@ mod tests { writer.append_int(0); // vref_array length writer.append_int(0); // jitcode_pos writer.append_int(0); // pc + writer.append_int(0); // py_pc writer.patch_current_size(0); let rd_numb = writer.create_numbering(); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs index f31ea892e23..8845f655be0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs @@ -26,6 +26,12 @@ fn test_outer_resume_jitcode_index() -> u32 { let mut pyjit = crate::PyJitCode::skeleton(std::ptr::null()); pyjit.jitcode = std::sync::Arc::new(runtime_jc); pyjit.metadata.is_drained = true; + // Forward `(jitcode_pc, py_pc)` resume markers a real drained JitCode + // carries: guard capture reads the innermost frame's Python pc forward + // from this table (never projected backward). The single-instruction + // synthetic body maps every JitCode offset to py_pc 0. + pyjit.metadata.forward_py_pc_marker_by_jit_pc = vec![(0, 0)]; + pyjit.metadata.forward_py_pc_pred_by_jit_pc = vec![(0, 0)]; let jitcode = crate::state::install_jitcode_for(std::ptr::null(), std::sync::Arc::new(pyjit)) as *const crate::state::JitCode; let index = unsafe { (*jitcode).index as u32 }; From 9f1bc8b0d0bdb4a19922fdf7486a655d269e2991 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2026 02:01:32 +0900 Subject: [PATCH 4/8] jit: address forward py_pc review findings (symmetric abort + dedup trivia skip) Two follow-ups from the Slice 3' code review of the forward py_pc plumbing: - trace_opcode.rs: `build_framestack_snapshot` requested a trace abort when the resume JitCode pc could not be resolved, but the paired forward Python-pc lookup silently fell back to the raw pc. A resolved JitCode pc with no forward marker now requests the abort too, matching both the sibling `pc` fallback and the production `forward_snapshot_py_pc` hard-fail, instead of publishing a fallback Python pc. - codewriter.rs: drop the duplicate `skip_python_trivia_forward` call in the marker-tier loop; the value is already bound for the forward-py_pc marker push and is reused for the depth-trivia marker. check.py: dynasm 285/285 + cranelift 285/285. Assisted-by: Claude --- pyre/pyre-jit-trace/src/trace_opcode.rs | 19 +++++++++++++++---- pyre/pyre-jit/src/jit/codewriter.rs | 2 -- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pyre/pyre-jit-trace/src/trace_opcode.rs b/pyre/pyre-jit-trace/src/trace_opcode.rs index 06344ffd49f..4f608c1f397 100644 --- a/pyre/pyre-jit-trace/src/trace_opcode.rs +++ b/pyre/pyre-jit-trace/src/trace_opcode.rs @@ -2913,10 +2913,21 @@ impl MIFrame { crate::state::request_trace_abort(); top_pc as u32 }); - let top_py_pc = resolved - .and_then(|offset| payload.resume_position_for_jitcode_pc(offset)) - .map(|(_, py_pc)| py_pc) - .unwrap_or(top_pc as u32); + let top_py_pc = match resolved { + // A resolved JitCode pc with no forward Python-pc marker cannot be + // resumed; decline the trace exactly like the `top_pc_word` arm + // above rather than silently publishing a fallback Python pc. + Some(offset) => payload + .resume_position_for_jitcode_pc(offset) + .map(|(_, py_pc)| py_pc) + .unwrap_or_else(|| { + crate::state::request_trace_abort(); + top_pc as u32 + }), + // `resolved` was None: the `top_pc_word` arm already requested the + // abort, so mirror its fallback without asking twice. + None => top_pc as u32, + }; let top_frame = majit_metainterp::recorder::SnapshotFrame { jitcode_index: top_jitcode_index, pc: top_pc_word, diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index 3ce0fbd012e..3963e01d480 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -13691,8 +13691,6 @@ impl CodeWriter { let skipped_py = pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py as usize); forward_py_pc_marker_by_jit_pc.push((off, skipped_py as u32)); - let skipped_py = - pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py as usize); let depth_trivia = static_depth.get(skipped_py).copied(); depth_trivia_marker_by_jit_pc.push((off, depth_trivia)); pcdep_trivia_marker_by_jit_pc.push(( From d495cbb5139caeb1ac7e14a0ca73306b458f7759 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2026 07:17:37 +0900 Subject: [PATCH 5/8] jit: source inlined-callee resumed-frame namespace from its own code globals build_resumed_frames assigned inner (non-outermost) ResumedFrame.namespace the chain virtualizable's vable_ns. Assign it the callee's own w_code globals via w_code_get_w_globals instead, falling back to vable_ns when the callee code carries no globals. Mirrors recover_inline_callee_globals. Assisted-by: Claude --- pyre/pyre-jit/src/eval.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index e97186fd8d8..1bc3dba9b7c 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -9506,8 +9506,22 @@ fn build_resumed_frames( } else { std::ptr::null() } + } else if !w_code.is_null() { + // get_w_globals: each inlined-callee section resolves LOAD_GLOBAL + // in its own module namespace, taken from its own code — not the + // chain virtualizable's (mirrors recover_inline_callee_globals). + // Fall back to the chain namespace only when the callee code + // carries no globals yet. + let callee_ns = unsafe { + pyre_interpreter::w_code_get_w_globals(w_code as pyre_object::PyObjectRef) + as *const () + }; + if !callee_ns.is_null() { + callee_ns + } else { + vable_ns + } } else { - // Inner frames share the chain virtualizable's namespace. vable_ns }; result.push(crate::call_jit::ResumedFrame { From e623a8010541cae72c7839958df0388c6ea39e5c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2026 07:17:42 +0900 Subject: [PATCH 6/8] jit: dedup op-start pred resume twins on jitcode offset via BTreeMap The six op-start-tier pred twins (forward_py_pc, depth_trivia, pcdep_trivia, const_ref_trivia, result_color_trivia, resume_marker) were built by pushing one entry per py_pc directly, leaving duplicate jitcode offsets whose binary-search winner depended on sort_unstable order. Build them from a BTreeMap (later py wins, matching by_off) so each offset yields one entry; drop the now redundant post-sorts. Duplicate py_pcs share skip_python_trivia_forward's target, so surviving values are unchanged (dynasm/cranelift 285/285). Assisted-by: Claude --- pyre/pyre-jit/src/jit/codewriter.rs | 80 +++++++++++++++-------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index 3963e01d480..a391aaa825c 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -13715,37 +13715,42 @@ impl CodeWriter { pcdep_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); const_ref_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_trivia_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); - // Op-start tier: predecessor scan, markers EXCLUDED. + // Op-start tier: predecessor scan, markers EXCLUDED. Dedup on the + // jitcode offset (later py wins, matching `by_off`): a run of + // trivia py_pcs collapses onto one jitcode offset, so direct-push + // left duplicate offsets whose binary-search winner depended on + // unstable-sort order. `BTreeMap` yields one offset-sorted entry + // per offset; duplicates share `skip_python_trivia_forward`'s + // target, so the surviving values are unchanged. + let mut pred_by_off: BTreeMap = BTreeMap::new(); for (py, &pos) in first_jit_pc_by_py_pc.iter().enumerate() { if pos != usize::MAX { - let skipped_py = - pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py); - forward_py_pc_pred_by_jit_pc.push((pos, skipped_py as u32)); - let depth_trivia = static_depth.get(skipped_py).copied(); - depth_trivia_pred_by_jit_pc.push((pos, depth_trivia)); - pcdep_trivia_pred_by_jit_pc.push(( - pos, - pcdep_color_slots - .get(skipped_py) - .cloned() - .unwrap_or_default(), - )); - const_ref_trivia_pred_by_jit_pc.push(( - pos, - const_ref_slots_at_pc - .get(skipped_py) - .cloned() - .unwrap_or_default(), - )); - let result_color_trivia = result_color_at_pc.get(skipped_py).copied(); - result_color_trivia_pred_by_jit_pc.push((pos, result_color_trivia)); + pred_by_off.insert(pos, py); } } - depth_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); - forward_py_pc_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); - pcdep_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); - const_ref_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); - result_color_trivia_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + for (&pos, &py) in pred_by_off.iter() { + let skipped_py = + pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py); + forward_py_pc_pred_by_jit_pc.push((pos, skipped_py as u32)); + let depth_trivia = static_depth.get(skipped_py).copied(); + depth_trivia_pred_by_jit_pc.push((pos, depth_trivia)); + pcdep_trivia_pred_by_jit_pc.push(( + pos, + pcdep_color_slots + .get(skipped_py) + .cloned() + .unwrap_or_default(), + )); + const_ref_trivia_pred_by_jit_pc.push(( + pos, + const_ref_slots_at_pc + .get(skipped_py) + .cloned() + .unwrap_or_default(), + )); + let result_color_trivia = result_color_at_pc.get(skipped_py).copied(); + result_color_trivia_pred_by_jit_pc.push((pos, result_color_trivia)); + } // Marker tier: exact-match, block-head precedence. for &(off, py) in &block_head_py_by_jit_pc { let skipped_py = @@ -13756,18 +13761,17 @@ impl CodeWriter { resume_marker_marker_by_jit_pc.push((off, marker)); } resume_marker_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); - // Op-start tier: predecessor scan, markers EXCLUDED. - for (py, &pos) in first_jit_pc_by_py_pc.iter().enumerate() { - if pos != usize::MAX { - let skipped_py = - pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py); - let marker = first_jit_pc_by_py_pc - .get(skipped_py) - .and_then(|_| resolve_marker(skipped_py)); - resume_marker_pred_by_jit_pc.push((pos, marker)); - } + // Op-start tier: predecessor scan, markers EXCLUDED. Reuse the + // deduped `pred_by_off` so the resume-marker twin matches the other + // op-start twins entry-for-entry (offset-sorted, no post-sort). + for (&pos, &py) in pred_by_off.iter() { + let skipped_py = + pyre_jit_trace::jitcode_dispatch::skip_python_trivia_forward(code, py); + let marker = first_jit_pc_by_py_pc + .get(skipped_py) + .and_then(|_| resolve_marker(skipped_py)); + resume_marker_pred_by_jit_pc.push((pos, marker)); } - resume_marker_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); // Marker tier: exact-match, block-head precedence. Compose the // runtime after-residual path's trivia skip and semantic // fallthrough before resolving the resume marker. From 7dae04a213a4b398d6cfd2f530a96e3a419356b7 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2026 11:05:40 +0900 Subject: [PATCH 7/8] jit: add after-residual fallthrough-py forward twin under audit (jitcode-blackhole Slice 4 Phase A) The inline-caller frame's after-residual-call resume python pc is computed at resume time by inverting call_jit_pc then taking semantic_fallthrough_pc. Build a codewriter twin after_residual_fallthrough_py_pc_{marker,pred}_by_jit_pc = semantic_fallthrough_pc(RAW resolving py), keyed by JitCode offset alongside depth_after_residual_* / result_color_after_residual_*, with accessor after_residual_fallthrough_py_pc_for_jitcode_pc and a _populated predicate. No consumer reads it yet; a PYRE_PCMAP_AFTERRESIDUAL_AUDIT assert at the inline-caller seam certifies twin == inverted-then-fallthrough. Byte-identical audit-off; dynasm/cranelift 285/285 both audit-off and audit-on. Assisted-by: Claude --- .../src/jitcode_dispatch/resume_snapshot.rs | 13 ++++++ pyre/pyre-jit-trace/src/pyjitcode.rs | 45 +++++++++++++++++++ pyre/pyre-jit-trace/src/state.rs | 2 + pyre/pyre-jit/src/jit/codewriter.rs | 12 +++++ 4 files changed, 72 insertions(+) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index fca08600265..561bc158660 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -1218,6 +1218,19 @@ pub(crate) fn compute_inline_caller_frame( )?; let code = &*jc.payload.code_ptr; let fallthrough = crate::pyjitpl::semantic_fallthrough_pc(code, call_py) as u32; + // #73 Slice 4 (twin-first): certify the forward `after_residual_fallthrough` + // twin reproduces the inverted-then-fallthrough coordinate before any + // consumer cuts over to it. + if pcmap_afterresidual_audit_enabled() + && jc.payload.after_residual_fallthrough_py_pc_populated() + { + assert_eq!( + jc.payload + .after_residual_fallthrough_py_pc_for_jitcode_pc(call_jit_pc), + Some(fallthrough), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: inline-caller fallthrough-py twin diverged at jit_pc {call_jit_pc} (call_py {call_py})" + ); + } ( jc.index as u32, fallthrough, diff --git a/pyre/pyre-jit-trace/src/pyjitcode.rs b/pyre/pyre-jit-trace/src/pyjitcode.rs index c9ad2f4d24d..c087aaa84cb 100644 --- a/pyre/pyre-jit-trace/src/pyjitcode.rs +++ b/pyre/pyre-jit-trace/src/pyjitcode.rs @@ -203,6 +203,14 @@ pub struct PyJitCodeMetadata { /// Empty for skeleton / fixture. pub depth_after_residual_marker_by_jit_pc: Vec<(usize, Option)>, pub depth_after_residual_pred_by_jit_pc: Vec<(usize, Option)>, + /// After-residual-call resume python pc for the inline-CALLER frame, keyed + /// by a JitCode byte offset with the same exact-marker / predecessor-op-start + /// split as `depth_after_residual_*`. Value = `semantic_fallthrough_pc(py)` + /// for the RAW resolving py — the coordinate the runtime otherwise obtains + /// by inverting `call_jit_pc` then taking the fallthrough. Empty for + /// skeleton / fixture. + pub after_residual_fallthrough_py_pc_marker_by_jit_pc: Vec<(usize, u32)>, + pub after_residual_fallthrough_py_pc_pred_by_jit_pc: Vec<(usize, u32)>, /// Whether codewriter register allocation assigned non-identity frame /// colors. Skeleton and portal metadata leave this false. pub has_color_map: bool, @@ -812,6 +820,41 @@ impl PyJitCode { || !self.metadata.depth_after_residual_pred_by_jit_pc.is_empty() } + /// After-residual-call resume python pc for the inline-caller frame, + /// resolved by JitCode byte offset with the same exact-marker / + /// predecessor-op-start tiers as `python_pc_for_jitcode_pc`. `None` when the + /// twin is empty (skeleton / fixture) — distinguish via + /// [`Self::after_residual_fallthrough_py_pc_populated`]. + pub fn after_residual_fallthrough_py_pc_for_jitcode_pc(&self, jit_pc: usize) -> Option { + let marker = &self + .metadata + .after_residual_fallthrough_py_pc_marker_by_jit_pc; + let pred = &self + .metadata + .after_residual_fallthrough_py_pc_pred_by_jit_pc; + if marker.is_empty() && pred.is_empty() { + return None; + } + if let Ok(i) = marker.binary_search_by_key(&jit_pc, |&(off, _)| off) { + return Some(marker[i].1); + } + let search = pred.binary_search_by_key(&jit_pc, |&(off, _)| off); + Self::predecessor_index(search).map(|i| pred[i].1) + } + + /// Whether the after-residual fallthrough-py twin carries entries. `false` + /// for skeleton / fixture installs. + pub fn after_residual_fallthrough_py_pc_populated(&self) -> bool { + !self + .metadata + .after_residual_fallthrough_py_pc_marker_by_jit_pc + .is_empty() + || !self + .metadata + .after_residual_fallthrough_py_pc_pred_by_jit_pc + .is_empty() + } + /// Post-`residual_call` catch resume marker keyed by a JitCode byte /// offset, resolved with the SAME exact-marker / predecessor-op-start /// tiers as `python_pc_for_jitcode_pc`. @@ -950,6 +993,8 @@ impl PyJitCode { result_color_after_residual_pred_by_jit_pc: Vec::new(), depth_after_residual_marker_by_jit_pc: Vec::new(), depth_after_residual_pred_by_jit_pc: Vec::new(), + after_residual_fallthrough_py_pc_marker_by_jit_pc: Vec::new(), + after_residual_fallthrough_py_pc_pred_by_jit_pc: Vec::new(), // Encoder/decoder readers in // `get_list_of_active_boxes`, `regalloc::external/input_indices`, // and `setup_bridge_sym::portal_red_regs_at` sentinel-skip both diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 2d816cf00d9..d89ddffc190 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -11648,6 +11648,8 @@ mod tests { result_color_after_residual_pred_by_jit_pc: Vec::new(), depth_after_residual_marker_by_jit_pc: Vec::new(), depth_after_residual_pred_by_jit_pc: Vec::new(), + after_residual_fallthrough_py_pc_marker_by_jit_pc: Vec::new(), + after_residual_fallthrough_py_pc_pred_by_jit_pc: Vec::new(), has_color_map: false, portal_frame_reg: 0, portal_ec_reg: 0, diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index a391aaa825c..8f2262d703b 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -13652,6 +13652,8 @@ impl CodeWriter { let mut result_color_after_residual_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut depth_after_residual_marker_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut depth_after_residual_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); + let mut after_residual_fallthrough_py_pc_marker_by_jit_pc: Vec<(usize, u32)> = Vec::new(); + let mut after_residual_fallthrough_py_pc_pred_by_jit_pc: Vec<(usize, u32)> = Vec::new(); let mut forward_py_pc_marker_by_jit_pc: Vec<(usize, u32)> = Vec::new(); let mut forward_py_pc_pred_by_jit_pc: Vec<(usize, u32)> = Vec::new(); let mut after_residual_call_resume_marker_by_jit_pc: Vec<(usize, Option)> = @@ -13790,10 +13792,16 @@ impl CodeWriter { result_color_after_residual_marker_by_jit_pc .push((off, result_color_at_pc.get(ft_rc).copied())); depth_after_residual_marker_by_jit_pc.push((off, static_depth.get(ft_rc).copied())); + // Same RAW-py fallthrough as result_color/depth: the caller + // frame's after-residual resume py = fallthrough(python pc of + // the call op). Carries the coordinate forward so the runtime + // read need not invert call_jit_pc. + after_residual_fallthrough_py_pc_marker_by_jit_pc.push((off, ft_rc as u32)); } after_residual_marker_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_after_residual_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); depth_after_residual_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + after_residual_fallthrough_py_pc_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); // Op-start tier: predecessor scan, markers EXCLUDED. for (py, &pos) in first_jit_pc_by_py_pc.iter().enumerate() { if pos != usize::MAX { @@ -13808,11 +13816,13 @@ impl CodeWriter { .push((pos, result_color_at_pc.get(ft_rc).copied())); depth_after_residual_pred_by_jit_pc .push((pos, static_depth.get(ft_rc).copied())); + after_residual_fallthrough_py_pc_pred_by_jit_pc.push((pos, ft_rc as u32)); } } after_residual_marker_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_after_residual_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); depth_after_residual_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + after_residual_fallthrough_py_pc_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); // Post-residual-call catch marker twin: source values from the // same sparse construction inputs, while resolving its key // with the exact block-head / predecessor-op-start split of the @@ -13887,6 +13897,8 @@ impl CodeWriter { result_color_after_residual_pred_by_jit_pc, depth_after_residual_marker_by_jit_pc, depth_after_residual_pred_by_jit_pc, + after_residual_fallthrough_py_pc_marker_by_jit_pc, + after_residual_fallthrough_py_pc_pred_by_jit_pc, has_color_map: !pcdep_color_slots.is_empty(), portal_frame_reg, portal_ec_reg, From 8d08ccde8e52f45f027df871f3917b0d6be25d4e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 23 Jul 2026 11:27:10 +0900 Subject: [PATCH 8/8] jit: cut after-residual fallthrough resume coordinate to the forward twin (jitcode-blackhole Slice 4 Phase B) resolve_parent_resume_py_pc's CallFallthrough arm and the self-recursive CALL_ASSEMBLER vstack seed computed the inline-caller frame's after-residual resume python pc by inverting call_jit_pc then taking semantic_fallthrough_pc. Read the forward after_residual_fallthrough_py_pc twin instead; the inversion survives only for the empty-twin class (populated code, no Python map) and as the PYRE_PCMAP_AFTERRESIDUAL_AUDIT oracle. The marker-miss depth raw() fallbacks stay Python-keyed (already covered by depth_after_residual). Byte-identical audit-off; dynasm/cranelift 285/285 both audit-off and audit-on. Assisted-by: Claude --- .../src/jitcode_dispatch/diag.rs | 28 ++++++++++++++-- .../src/jitcode_dispatch/inline_call.rs | 33 +++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs index ee84a11f9d0..ea14ad0f2ec 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs @@ -71,9 +71,31 @@ pub(crate) fn resolve_parent_resume_py_pc(parent: &InlineParentFrame) -> Option< if pjc.code_ptr.is_null() { return None; } - let call_py_pc = python_pc_for_jitcode_pc(&pjc.metadata, call_jit_pc) as usize; - let code = unsafe { &*pjc.code_ptr }; - Some(crate::pyjitpl::semantic_fallthrough_pc(code, call_py_pc) as u32) + // #73 Slice 4: read the forward after-residual fallthrough twin. The + // inversion survives only for the empty-twin class (populated code + // with no Python map) and as the audit oracle. + let legacy = || { + let call_py_pc = python_pc_for_jitcode_pc(&pjc.metadata, call_jit_pc) as usize; + let code = unsafe { &*pjc.code_ptr }; + crate::pyjitpl::semantic_fallthrough_pc(code, call_py_pc) as u32 + }; + let twin = pjc + .after_residual_fallthrough_py_pc_populated() + .then(|| pjc.after_residual_fallthrough_py_pc_for_jitcode_pc(call_jit_pc)) + .flatten(); + match twin { + Some(ft) => { + if pcmap_afterresidual_audit_enabled() { + assert_eq!( + ft, + legacy(), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: parent-resume fallthrough-py twin diverged at jit_pc {call_jit_pc}" + ); + } + Some(ft) + } + None => Some(legacy()), + } } } } diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 1d846721d59..2cd8f3b4b4c 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -925,8 +925,37 @@ pub(crate) fn try_walker_call_assembler_self_recursive( if ctx.vstack_valid { let caller_jitcode = unsafe { &*sym.jitcode() }; let caller_code = unsafe { &*caller_jitcode.payload.code_ptr }; - let call_py_pc = python_pc_for_jitcode_pc(&caller_jitcode.payload.metadata, op.pc) as usize; - let resume_py_pc = crate::pyjitpl::semantic_fallthrough_pc(caller_code, call_py_pc) as u32; + // #73 Slice 4: forward after-residual fallthrough coordinate; the + // inversion survives only for the empty-twin class and as the audit + // oracle. + let legacy_resume_py_pc = || { + let call_py_pc = + python_pc_for_jitcode_pc(&caller_jitcode.payload.metadata, op.pc) as usize; + crate::pyjitpl::semantic_fallthrough_pc(caller_code, call_py_pc) as u32 + }; + let resume_py_pc = match caller_jitcode + .payload + .after_residual_fallthrough_py_pc_populated() + .then(|| { + caller_jitcode + .payload + .after_residual_fallthrough_py_pc_for_jitcode_pc(op.pc) + }) + .flatten() + { + Some(ft) => { + if pcmap_afterresidual_audit_enabled() { + assert_eq!( + ft, + legacy_resume_py_pc(), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: self-recursive CA vstack fallthrough-py twin diverged at jit_pc {}", + op.pc + ); + } + ft + } + None => legacy_resume_py_pc(), + }; let raw_depth = || { crate::liveness::liveness_for(caller_jitcode.payload.code_ptr) .depth_at_py_pc()