jit: count eval-breaker back-edge poll failures separately

majit/majit-backend/src/resume_guard_descr.rs

@@ -39,7 +39,7 @@
 use std::any::Any;
 use std::cell::UnsafeCell;
 use std::rc::Rc;
-use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering};
 use std::sync::{Arc, OnceLock};
 
 use majit_ir::{
@@ -167,6 +167,12 @@ pub struct ResumeGuardDescr {
     /// the meta Arc is the single source of truth.  Sorted and deduped
     /// at write time so `is_force_token_slot` can use `binary_search`.
     pub force_token_slots: UnsafeCell<Vec<usize>>,
+    /// This guard is the eval-breaker word's back-edge poll, not a check on
+    /// traced values.  Stamped once per emission by the optimizer, which is
+    /// where the guard's condition chain is still in hand; read by the
+    /// statistics counter to keep scheduled exits out of the guard-failure
+    /// total.  See `descr.rs FailDescr::is_back_edge_poll`.
+    pub back_edge_poll: AtomicBool,
     /// `AbstractResumeGuardDescr.handle_fail` (`compile.py:701-717`)
     /// drives `must_compile` via `jitcounter.tick(status_hash)` in
     /// RPython.  Pyre keeps a raw per-descr counter:
@@ -302,6 +308,7 @@ impl Descr for ResumeGuardDescr {
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: AtomicPtr::new(std::ptr::null_mut()),
             external_jump_target: OnceLock::new(),
@@ -447,6 +454,12 @@ impl FailDescr for ResumeGuardDescr {
         // Safety: single-threaded JIT.
         unsafe { *self.source_op_index.get() = Some(source_op_index) };
     }
+    fn is_back_edge_poll(&self) -> bool {
+        self.back_edge_poll.load(Ordering::Relaxed)
+    }
+    fn set_back_edge_poll(&self) {
+        self.back_edge_poll.store(true, Ordering::Relaxed);
+    }
     fn force_token_slots(&self) -> Vec<usize> {
         // Safety: single-threaded JIT.
         unsafe { (&*self.force_token_slots.get()).clone() }
@@ -578,6 +591,7 @@ pub fn make_resume_guard_descr_typed(types: Vec<Type>) -> DescrRef {
         fail_index_per_trace: AtomicU32::new(0),
         source_op_index: UnsafeCell::new(None),
         force_token_slots: UnsafeCell::new(Vec::new()),
+        back_edge_poll: AtomicBool::new(false),
         fail_count: AtomicU32::new(0),
         trace_info: AtomicPtr::new(std::ptr::null_mut()),
         external_jump_target: OnceLock::new(),

majit/majit-ir/src/descr.rs

@@ -4055,6 +4055,30 @@ pub trait FailDescr: Descr {
         );
     }
 
+    /// Pyre-only per-emission slot: this guard is the eval-breaker word's
+    /// back-edge poll rather than a check on traced values.
+    ///
+    /// Its failures are scheduled exits — the collector armed the word and the
+    /// loop has to leave machine code for the request to be serviced — so they
+    /// describe when a collection landed, not what the compiled code assumed.
+    /// The statistics counter reads this to keep the two apart
+    /// (`crate::eval_breaker_word::is_back_edge_poll_guard` decides it).
+    /// Default `false` for non-resume FailDescrs.
+    fn is_back_edge_poll(&self) -> bool {
+        false
+    }
+
+    /// Pyre-only per-emission slot write.  See `is_back_edge_poll`.  Default
+    /// panics — only Resume-family guards own the slot.  Callers must gate by
+    /// `is_resume_guard() || is_resume_guard_copied()`.
+    fn set_back_edge_poll(&self) {
+        panic!(
+            "set_back_edge_poll invoked on a FailDescr that does not \
+             carry the per-emission back_edge_poll slot (only \
+             ResumeGuardDescr / ResumeGuardCopiedDescr own it)"
+        );
+    }
+
     /// `compile.py:683` `AbstractResumeGuardDescr._attrs_ = ('status',)`
     /// — packs `ST_BUSY_FLAG` + type tag + hash on the resume-guard
     /// descr.  `compile.py:741-745` `self.status` read for

majit/majit-ir/src/eval_breaker_word.rs

@@ -136,6 +136,52 @@ pub fn take_gc() -> bool {
     EVAL_BREAKER_WORD.fetch_and(!EB_GC, Ordering::Relaxed) & EB_GC != 0
 }
 
+/// Depth of the operation chain between the poll's load and its guard.
+///
+/// The recorder emits `RawLoadI -> IntAnd -> IntIsTrue -> GuardFalse`, so two
+/// links separate the guard's condition from the load. The walk below allows
+/// a few more so that an optimizer pass inserting or splitting one link does
+/// not silently stop matching, and stops well before a long pure chain.
+const POLL_CHAIN_DEPTH: usize = 6;
+
+/// Whether `guard`'s condition is a back-edge poll of this word.
+///
+/// The poll is recorded as
+/// `GuardFalse(IntIsTrue(IntAnd(RawLoadI(addr, 0), JIT_BREAKER_MASK)))`. The
+/// match anchors on the `RawLoadI` of the published address rather than on the
+/// whole shape: that load is the one link the recorder cannot drop (it must
+/// stay non-pure and outside the always-pure range, or CSE forwards the
+/// preamble's guarded-zero value into the loop body), so walking back to it
+/// survives rewrites of the links in between.
+///
+/// An unpublished address reads `0` and no poll is recorded, so the walk
+/// declines rather than matching an unrelated load from a null constant.
+pub fn is_back_edge_poll_guard(guard: &crate::resoperation::Op) -> bool {
+    let addr = eval_breaker_word_addr();
+    if addr == 0 {
+        return false;
+    }
+    // Control-flow guards (`GUARD_NOT_FORCED`, `GUARD_NO_EXCEPTION`, ...) carry
+    // no condition operand at all.
+    if guard.num_args() == 0 {
+        return false;
+    }
+    let mut operand = guard.arg(0);
+    for _ in 0..POLL_CHAIN_DEPTH {
+        let Some(op) = operand.bound_op() else {
+            return false;
+        };
+        if op.num_args() == 0 {
+            return false;
+        }
+        if op.opcode == crate::resoperation::OpCode::RawLoadI {
+            return op.arg(0).const_int() == Some(addr as i64);
+        }
+        operand = op.arg(0);
+    }
+    false
+}
+
 /// Every flag must fit in the word the poll actually loads. Checked per target,
 /// so a flag too wide for a 32-bit `usize` fails the wasm32 build rather than
 /// silently reading as unarmed there.
@@ -151,6 +197,63 @@ const _: () = assert!(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::resoperation::{Op, OpCode};
+    use crate::value::Const;
+    use std::rc::Rc;
+
+    fn bind(op: Op) -> crate::operand::Operand {
+        crate::operand::Operand::from_bound_op(&Rc::new(op))
+    }
+
+    fn int(value: i64) -> crate::operand::Operand {
+        crate::operand::Operand::const_(Const::Int(value))
+    }
+
+    /// Build the recorded back-edge poll over `load_addr`, which the caller
+    /// varies to separate "this word" from "some other raw load".
+    fn poll_guard(load_addr: usize) -> Op {
+        let load = bind(Op::new(OpCode::RawLoadI, &[int(load_addr as i64), int(0)]));
+        let masked = bind(Op::new(
+            OpCode::IntAnd,
+            &[load, int(JIT_BREAKER_MASK as i64)],
+        ));
+        let armed = bind(Op::new(OpCode::IntIsTrue, &[masked]));
+        Op::new(OpCode::GuardFalse, &[armed])
+    }
+
+    /// The counter split turns on this predicate alone, so it has to separate
+    /// the poll from every other guard the same trace records. Only the
+    /// address published for this word matches: a raw load of a neighbouring
+    /// address reaches the same opcode chain, and a data guard reaches none of
+    /// it.
+    ///
+    /// `publish_addr` is idempotent and writes only the address holder, not the
+    /// word, so this leaves the process-global flag state alone.
+    #[test]
+    fn only_a_poll_of_this_words_address_is_recognised() {
+        publish_addr();
+        let addr = eval_breaker_word_addr();
+        assert_ne!(addr, 0, "publish_addr must make the address readable");
+
+        assert!(is_back_edge_poll_guard(&poll_guard(addr)));
+        // Same shape, a different word: the JIT records other raw loads.
+        assert!(!is_back_edge_poll_guard(&poll_guard(
+            addr + EVAL_BREAKER_WORD_SIZE
+        )));
+        // A guard on traced values — the population whose failures the
+        // `guard_failures` total is meant to describe.
+        let value = bind(Op::new(OpCode::IntAdd, &[int(1), int(2)]));
+        let is_true = bind(Op::new(OpCode::IntIsTrue, &[value]));
+        assert!(!is_back_edge_poll_guard(&Op::new(
+            OpCode::GuardTrue,
+            &[is_true]
+        )));
+        // A control-flow guard carries no condition operand at all.
+        assert!(!is_back_edge_poll_guard(&Op::new(
+            OpCode::GuardNotForced,
+            &[]
+        )));
+    }
 
     /// One request arms the poll once: the taker reports it, clears it, and the
     /// next taker sees nothing. A taker that failed to clear would keep failing

majit/majit-metainterp/src/compile.rs

@@ -3008,6 +3008,7 @@ pub fn make_fail_descr_with_index(fail_index: u32, num_live: usize) -> DescrRef
         fail_index_per_trace: AtomicU32::new(0),
         source_op_index: UnsafeCell::new(None),
         force_token_slots: UnsafeCell::new(Vec::new()),
+        back_edge_poll: std::sync::atomic::AtomicBool::new(false),
         fail_count: AtomicU32::new(0),
         trace_info: AtomicPtr::new(std::ptr::null_mut()),
         external_jump_target: OnceLock::new(),
@@ -3100,6 +3101,7 @@ pub fn make_resume_guard_descr_typed(types: Vec<Type>) -> DescrRef {
         fail_index_per_trace: AtomicU32::new(0),
         source_op_index: UnsafeCell::new(None),
         force_token_slots: UnsafeCell::new(Vec::new()),
+        back_edge_poll: std::sync::atomic::AtomicBool::new(false),
         fail_count: AtomicU32::new(0),
         trace_info: AtomicPtr::new(std::ptr::null_mut()),
         external_jump_target: OnceLock::new(),
@@ -3380,6 +3382,7 @@ pub fn make_resume_at_position_descr_typed(types: Vec<Type>) -> DescrRef {
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: std::sync::atomic::AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: AtomicPtr::new(std::ptr::null_mut()),
             external_jump_target: OnceLock::new(),
@@ -3647,6 +3650,7 @@ pub fn make_resume_guard_forced_descr_typed(types: Vec<Type>) -> DescrRef {
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: std::sync::atomic::AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: AtomicPtr::new(std::ptr::null_mut()),
             external_jump_target: OnceLock::new(),
@@ -3898,6 +3902,7 @@ pub fn make_resume_guard_exc_descr_typed(types: Vec<Type>) -> DescrRef {
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: std::sync::atomic::AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: AtomicPtr::new(std::ptr::null_mut()),
             external_jump_target: OnceLock::new(),
@@ -3988,6 +3993,11 @@ pub struct ResumeGuardCopiedDescr {
     /// lives on the descr.  Same per-emission scoping as
     /// `source_op_index` / `rd_locs` — owned per copied descr.
     force_token_slots: UnsafeCell<Vec<usize>>,
+    /// Pyre-only per-emission slot: this guard is the eval-breaker word's
+    /// back-edge poll.  Same per-emission scoping as `source_op_index`; a
+    /// copy guards the same back edge as its donor, so the optimizer stamps
+    /// each copy as it emits it.
+    back_edge_poll: std::sync::atomic::AtomicBool,
     /// Pyre-only per-emission failure counter for bridge compilation
     /// thresholds.  PyPy carries the equivalent jitcounter hash in
     /// `compile.py:683 AbstractResumeGuardDescr._attrs_ ('status',)`
@@ -4134,6 +4144,7 @@ impl majit_ir::Descr for ResumeGuardCopiedDescr {
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: std::sync::atomic::AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
             bridge_code_ptr_cache: Box::new(std::sync::atomic::AtomicUsize::new(0)),
@@ -4312,6 +4323,26 @@ impl FailDescr for ResumeGuardCopiedDescr {
     fn set_source_op_index(&self, source_op_index: usize) {
         unsafe { *self.source_op_index.get() = Some(source_op_index) };
     }
+    /// Per-emission like `force_token_slots` below, and deliberately NOT
+    /// chased through `prev`: the classification describes this guard's own
+    /// condition chain, which a sharer does not inherit from its donor.  A
+    /// copied descr always answers `false`, and that is correct twice over.
+    /// A poll guard can never be the *sharer*: sharing requires
+    /// `!op.has_descr() && op.rd_resume_position < 0` (optimizer.rs
+    /// `emit_guard_operation`, mirrored in optimizeopt/mod.rs), whereas the
+    /// poll is emitted by `close_loop_args_at` through `generate_guard`,
+    /// which captures resume data and so always carries a resume position.
+    /// And when a poll guard is the *donor*, the sharer is some descrless
+    /// optimizer-created follow-up guard that is not itself a poll, so
+    /// reading through `prev` would misreport it as one.
+    fn is_back_edge_poll(&self) -> bool {
+        self.back_edge_poll
+            .load(std::sync::atomic::Ordering::Relaxed)
+    }
+    fn set_back_edge_poll(&self) {
+        self.back_edge_poll
+            .store(true, std::sync::atomic::Ordering::Relaxed);
+    }
     /// Per-emission `force_token_slots` (see field comment).  Owned
     /// per copied descr so each emission's GC-root classification
     /// stays distinct — PyPy bakes the equivalent map inline per
@@ -4450,6 +4481,7 @@ impl majit_ir::Descr for ResumeGuardCopiedExcDescr {
                 fail_index_per_trace: AtomicU32::new(0),
                 source_op_index: UnsafeCell::new(None),
                 force_token_slots: UnsafeCell::new(Vec::new()),
+                back_edge_poll: std::sync::atomic::AtomicBool::new(false),
                 fail_count: AtomicU32::new(0),
                 trace_info: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
                 bridge_code_ptr_cache: Box::new(std::sync::atomic::AtomicUsize::new(0)),
@@ -4571,6 +4603,12 @@ impl FailDescr for ResumeGuardCopiedExcDescr {
     fn set_source_op_index(&self, source_op_index: usize) {
         self.inner.set_source_op_index(source_op_index);
     }
+    fn is_back_edge_poll(&self) -> bool {
+        self.inner.is_back_edge_poll()
+    }
+    fn set_back_edge_poll(&self) {
+        self.inner.set_back_edge_poll();
+    }
     fn force_token_slots(&self) -> Vec<usize> {
         self.inner.force_token_slots()
     }
@@ -4654,6 +4692,7 @@ pub fn make_resume_guard_copied_descr(prev: DescrRef) -> DescrRef {
         fail_index_per_trace: AtomicU32::new(0),
         source_op_index: UnsafeCell::new(None),
         force_token_slots: UnsafeCell::new(Vec::new()),
+        back_edge_poll: std::sync::atomic::AtomicBool::new(false),
         fail_count: AtomicU32::new(0),
         trace_info: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
         bridge_code_ptr_cache: Box::new(std::sync::atomic::AtomicUsize::new(0)),
@@ -4694,6 +4733,7 @@ pub fn make_resume_guard_copied_exc_descr(prev: DescrRef) -> DescrRef {
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: std::sync::atomic::AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()),
             bridge_code_ptr_cache: Box::new(std::sync::atomic::AtomicUsize::new(0)),
@@ -4858,6 +4898,7 @@ impl majit_ir::Descr for CompileLoopVersionDescr {
                 fail_index_per_trace: AtomicU32::new(0),
                 source_op_index: UnsafeCell::new(None),
                 force_token_slots: UnsafeCell::new(Vec::new()),
+                back_edge_poll: std::sync::atomic::AtomicBool::new(false),
                 fail_count: AtomicU32::new(0),
                 trace_info: AtomicPtr::new(std::ptr::null_mut()),
                 external_jump_target: OnceLock::new(),
@@ -5078,6 +5119,7 @@ fn make_compile_loop_version_descr_with_payload(types: Vec<Type>, payload: RdPay
             fail_index_per_trace: AtomicU32::new(0),
             source_op_index: UnsafeCell::new(None),
             force_token_slots: UnsafeCell::new(Vec::new()),
+            back_edge_poll: std::sync::atomic::AtomicBool::new(false),
             fail_count: AtomicU32::new(0),
             trace_info: AtomicPtr::new(std::ptr::null_mut()),
             external_jump_target: OnceLock::new(),
@@ -5580,6 +5622,7 @@ mod fail_descr_tests {
                 fail_index_per_trace: AtomicU32::new(0),
                 source_op_index: UnsafeCell::new(None),
                 force_token_slots: UnsafeCell::new(Vec::new()),
+                back_edge_poll: std::sync::atomic::AtomicBool::new(false),
                 fail_count: AtomicU32::new(0),
                 trace_info: AtomicPtr::new(std::ptr::null_mut()),
                 external_jump_target: OnceLock::new(),

majit/majit-metainterp/src/optimizeopt/mod.rs

@@ -6529,6 +6529,16 @@ impl OptContext {
             fd.set_rd_consts(Some(rd_consts));
             fd.set_rd_virtuals(descr_rd_virtuals);
             fd.set_rd_pendingfields(descr_pending);
+            // The back-edge poll of the eval-breaker word reaches this
+            // function like any other guard, and this is the last point where
+            // its condition chain and its descr are both in hand — the
+            // backends see only the emitted guard.  Stamping here also covers
+            // every re-emission (unroll's preamble and body copies, and the
+            // poll a bridge records at its own back edge), because each mints
+            // a fresh descr through this same call.
+            if majit_ir::eval_breaker_word::is_back_edge_poll_guard(op) {
+                fd.set_back_edge_poll();
+            }
         }
         // resume.py: RPython does NOT carry frame sizes out-of-band.
         // The decoder reads jitcode liveness (jitcode.position_info) at

majit/majit-metainterp/src/optimizeopt/optimizer.rs

@@ -2106,8 +2106,11 @@ impl Optimizer {
     /// failing a guard has no back edge and runs zero iterations while
     /// reporting a healthy-looking op total.
     ///
-    /// `label` names which of the three compile paths produced the trace, since
-    /// they differ in whether the body was unrolled.
+    /// `label` names which compile path produced the trace, since they differ
+    /// in whether the body was unrolled and whether it is a bridge. Bridges
+    /// must be logged too: they take trace ids from the same counter the loops
+    /// draw from, so omitting them leaves gaps that make a dumped trace
+    /// impossible to line up with the `trace=` field of a guard-failure log.
     pub fn log_optimized_trace<V, T, C>(label: &str, ops: &[T], constants: &C)
     where
         V: std::fmt::Debug,