jit: overflow-arm specialization, paused-caller resume PC fix, dynasm GuardOverflow CC, attr live markers

majit/majit-backend-dynasm/src/aarch64/assembler.rs

@@ -3490,7 +3490,20 @@ impl<'a> AssemblerARM64<'a> {
                 self.emit_guard_no_exception_check();
                 self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs);
             }
-            OpCode::GuardNoOverflow | OpCode::GuardOverflow => {
+            OpCode::GuardNoOverflow => {
+                self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs);
+            }
+            OpCode::GuardOverflow => {
+                // aarch64/opassembler.py:547-551 aliases GUARD_NO_OVERFLOW to
+                // guard_true and GUARD_OVERFLOW to guard_false. The overflow
+                // arithmetic producer leaves the no-overflow success CC in
+                // `guard_success_cc`, so invert it for the expected-overflow
+                // arm before the common guard emitter derives its fail CC.
+                let no_overflow_cc = self
+                    .guard_success_cc
+                    .take()
+                    .expect("GuardOverflow requires a preceding overflow operation");
+                self.guard_success_cc = Some(invert_cc(no_overflow_cc));
                 self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs);
             }
             OpCode::GuardNotForced | OpCode::GuardNotForced2 => {

majit/majit-backend-dynasm/src/j2plan.rs

@@ -165,24 +165,15 @@ pub(crate) struct TracePlan {
     pub inputargs: Vec<OpRef>,
     pub ops: Vec<LirOp>,
     pub live_points: Vec<LivePoint>,
-    pub deopt_spill_points: Vec<DeoptSpillPoint>,
     pub max_live: usize,
     pub lowered_ops: usize,
     pub fallback_ops: usize,
 }
 
-/// Guard fail args that are only needed on the deopt path at this point.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct DeoptSpillPoint {
-    pub op_index: usize,
-    pub args: Vec<OpRef>,
-}
-
 impl TracePlan {
     pub(crate) fn build(inputargs: &[InputArg], ops: &[Op]) -> Self {
         let lowered: Vec<LirOp> = ops.iter().map(|op| lower_op(op)).collect();
         let live_points = compute_live_points(&lowered);
-        let deopt_spill_points = compute_deopt_spill_points(&lowered);
         let max_live = live_points
             .iter()
             .map(|point| point.live_in.len())
@@ -196,24 +187,13 @@ impl TracePlan {
             fallback_ops,
             ops: lowered,
             live_points,
-            deopt_spill_points,
             max_live,
         }
     }
 
     pub(crate) fn summary(&self) -> TracePlanSummary<'_> {
         TracePlanSummary(self)
     }
-
-    pub(crate) fn deopt_spill_args_by_index(&self, len: usize) -> Vec<Vec<OpRef>> {
-        let mut by_index = vec![Vec::new(); len];
-        for point in &self.deopt_spill_points {
-            if point.op_index < by_index.len() {
-                by_index[point.op_index] = point.args.clone();
-            }
-        }
-        by_index
-    }
 }
 
 pub(crate) struct TracePlanSummary<'a>(&'a TracePlan);
@@ -223,12 +203,11 @@ impl fmt::Display for TracePlanSummary<'_> {
         let plan = self.0;
         write!(
             f,
-            "ops={} lowered={} fallback={} max_live={} deopt_spills={}",
+            "ops={} lowered={} fallback={} max_live={}",
             plan.ops.len(),
             plan.lowered_ops,
             plan.fallback_ops,
             plan.max_live,
-            plan.deopt_spill_points.len()
         )
     }
 }
@@ -400,33 +379,6 @@ fn compute_live_points(ops: &[LirOp]) -> Vec<LivePoint> {
     points
 }
 
-fn compute_deopt_spill_points(ops: &[LirOp]) -> Vec<DeoptSpillPoint> {
-    let mut fast_live_after = Vec::new();
-    let mut points = Vec::new();
-
-    for (op_index, op) in ops.iter().enumerate().rev() {
-        if let LirOp::Guard { fail_args, .. } = op {
-            let mut args = Vec::new();
-            for &arg in fail_args {
-                if !fast_live_after.contains(&arg) {
-                    add_ref(&mut args, arg);
-                }
-            }
-            if !args.is_empty() {
-                points.push(DeoptSpillPoint { op_index, args });
-            }
-        }
-
-        if let Some(dst) = op.def() {
-            remove_ref(&mut fast_live_after, dst);
-        }
-        op.add_uses(&mut fast_live_after);
-    }
-
-    points.reverse();
-    points
-}
-
 fn int_bin_kind(opcode: OpCode) -> IntBinKind {
     match opcode {
         OpCode::IntAdd => IntBinKind::Add,
@@ -665,7 +617,6 @@ mod tests {
         );
 
         assert_eq!(plan.fallback_ops, 0);
-        assert!(plan.deopt_spill_points.is_empty());
         assert!(matches!(
             plan.ops[1],
             LirOp::IntBin {
@@ -715,44 +666,11 @@ mod tests {
         assert!(guard_live.contains(&OpRef::int_op(1)));
         assert!(guard_live.contains(&OpRef::int_op(2)));
 
-        assert_eq!(
-            plan.deopt_spill_points,
-            vec![super::DeoptSpillPoint {
-                op_index: 2,
-                args: vec![OpRef::int_op(1)]
-            }]
-        );
-
         let add_live = &plan.live_points[0].live_in;
         assert!(add_live.contains(&i0));
         assert!(!add_live.contains(&c1));
     }
 
-    #[test]
-    fn deopt_spill_point_keeps_jump_args_on_fast_path() {
-        let i0 = OpRef::int_op(0);
-        let c1 = OpRef::const_int(1);
-
-        let add = Op::new(OpCode::IntAdd, &[rb(i0), rb(c1)]);
-        add.pos.set(OpRef::int_op(1));
-
-        let is_true = Op::new(OpCode::IntIsTrue, &[rb(OpRef::int_op(1))]);
-        is_true.pos.set(OpRef::int_op(2));
-
-        let guard = Op::new(OpCode::GuardTrue, &[rb(OpRef::int_op(2))]);
-        guard.pos.set(OpRef::int_op(3));
-        guard.setfailargs(vec![rb(OpRef::int_op(1))].into());
-        let jump = Op::new(OpCode::Jump, &[rb(OpRef::int_op(1))]);
-        jump.pos.set(OpRef::int_op(4));
-
-        let plan = TracePlan::build(
-            &[InputArg::from_type(Type::Int, 0)],
-            &[add, is_true, guard, jump],
-        );
-
-        assert!(plan.deopt_spill_points.is_empty());
-    }
-
     #[test]
     fn lowers_indexed_memory_operands_by_role() {
         let base = OpRef::int_op(0);

majit/majit-backend-dynasm/src/regalloc.rs

@@ -1692,10 +1692,6 @@ pub struct RegAlloc<'a> {
     /// closing JUMP, used by `_compute_hint_locations_from_descr` to pin
     /// reg hints via `longevity.fixed_register(position, reg, box)`.
     final_jump_op_position: i32,
-    /// j2-style deopt-only fail args, indexed by original operation index.
-    /// These values are needed by guard recovery but not by the fast path
-    /// after the guard, so keeping them in registers only increases pressure.
-    j2_deopt_spill_args: Vec<Vec<OpRef>>,
     /// j2-lowered operations, indexed by original operation index. The main
     /// dispatch path consumes these; legacy opcode dispatch is a guard rail
     /// only if a plan entry is missing.
@@ -1736,7 +1732,6 @@ impl<'a> RegAlloc<'a> {
             jump_target_descr: None,
             final_jump_args: None,
             final_jump_op_position: -1,
-            j2_deopt_spill_args: Vec::new(),
             j2_ops: Vec::new(),
             temp_var_counter: 0,
         }
@@ -1829,7 +1824,6 @@ impl<'a> RegAlloc<'a> {
         self.final_jump_args = None;
         self.final_jump_op_position = -1;
         let j2_plan = crate::j2plan::TracePlan::build(self.inputargs, self.operations);
-        self.j2_deopt_spill_args = j2_plan.deopt_spill_args_by_index(self.operations.len());
         self.j2_ops = j2_plan.ops;
         // x86/regalloc.py:191 X86RegisterHints().add_hints(longevity, inputargs, operations)
         //
@@ -2246,7 +2240,6 @@ impl<'a> RegAlloc<'a> {
         result_loc: Option<Loc>,
         output: &mut Vec<RegAllocOp>,
     ) {
-        self.spill_j2_deopt_args(op_index);
         self.flush_moves(output);
         let faillocs = self.locs_for_fail(op);
         output.push(RegAllocOp::PerformGuard {
@@ -2265,7 +2258,6 @@ impl<'a> RegAlloc<'a> {
         result_loc: Option<Loc>,
         output: &mut Vec<RegAllocOp>,
     ) {
-        self.spill_j2_deopt_args(op_index);
         self.flush_moves(output);
         let faillocs = self.locs_for_fail_args(fail_args);
         output.push(RegAllocOp::PerformGuard {
@@ -2276,56 +2268,6 @@ impl<'a> RegAlloc<'a> {
         });
     }
 
-    fn spill_j2_deopt_args(&mut self, op_index: usize) {
-        let Some(args) = self.j2_deopt_spill_args.get(op_index).cloned() else {
-            return;
-        };
-        for arg in args {
-            if arg.is_none() || arg.is_constant() {
-                continue;
-            }
-            let tp = self.tp(arg);
-            if tp == Type::Float {
-                Self::spill_deopt_arg_from_manager(
-                    &mut self.xrm,
-                    arg,
-                    tp,
-                    &mut self.longevity,
-                    &mut self.fm,
-                );
-            } else {
-                Self::spill_deopt_arg_from_manager(
-                    &mut self.rm,
-                    arg,
-                    tp,
-                    &mut self.longevity,
-                    &mut self.fm,
-                );
-            }
-        }
-    }
-
-    fn spill_deopt_arg_from_manager(
-        mgr: &mut RegisterManager,
-        arg: OpRef,
-        tp: Type,
-        longevity: &mut LifetimeManager,
-        fm: &mut FrameManager,
-    ) {
-        let Some(reg) = mgr.reg_bindings_get(arg, longevity) else {
-            return;
-        };
-        mgr._sync_var_to_stack(arg, tp, longevity, fm);
-        mgr.reg_bindings_del(arg, longevity);
-        mgr.free_regs.push(reg);
-        if crate::majit_log_enabled() {
-            eprintln!(
-                "[dynasm:j2plan] spill deopt-only failarg {:?} from {:?}",
-                arg, reg
-            );
-        }
-    }
-
     /// aarch64/regalloc.py:1089 get_gcmap.
     pub fn get_gcmap(&self, forbidden_regs: &[RegLoc], noregs: bool) -> *mut usize {
         let frame_depth = self.fm.get_frame_depth();
@@ -6321,7 +6263,13 @@ mod tests {
     }
 
     #[test]
-    fn test_j2_deopt_only_failarg_spilled_before_guard() {
+    fn test_j2_deopt_only_failarg_kept_in_register_at_guard() {
+        // A deopt-only fail arg (dead on the fast path after the guard) is
+        // captured from its register at the guard, not eagerly spilled to a
+        // frame slot: the failure path saves every register before rebuilding
+        // the frame, so a register faillocs is fully recoverable, and the
+        // fast path pays no spill store.
+        //
         // i0 is the typed inputarg slot 0 (Int) that the regalloc
         // registers from `inputargs.opref()`. i1/i2 are op result
         // positions; their variant tag is unconstrained, so plain
@@ -6368,8 +6316,8 @@ mod tests {
             panic!("guard op was not lowered through RegAllocOp::PerformGuard");
         };
         assert!(
-            matches!(faillocs.as_slice(), [Some(Loc::Frame(_))]),
-            "deopt-only failarg should be captured from a frame slot: {:?}",
+            matches!(faillocs.as_slice(), [Some(Loc::Reg(_))]),
+            "deopt-only failarg should be captured from a register: {:?}",
             faillocs
         );
     }

majit/majit-backend-dynasm/src/x86/assembler.rs

@@ -4651,7 +4651,20 @@ impl<'a> Assembler386<'a> {
                 self.emit_guard_no_exception_check();
                 self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs);
             }
-            OpCode::GuardNoOverflow | OpCode::GuardOverflow => {
+            OpCode::GuardNoOverflow => {
+                self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs);
+            }
+            OpCode::GuardOverflow => {
+                // x86/assembler.py:1873-1874 aliases GUARD_NO_OVERFLOW to
+                // guard_true and GUARD_OVERFLOW to guard_false. The overflow
+                // arithmetic producer leaves the no-overflow success CC in
+                // `guard_success_cc`, so invert it for the expected-overflow
+                // arm before the common guard emitter derives its fail CC.
+                let no_overflow_cc = self
+                    .guard_success_cc
+                    .take()
+                    .expect("GuardOverflow requires a preceding overflow operation");
+                self.guard_success_cc = Some(invert_cc(no_overflow_cc));
                 self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs);
             }
             OpCode::GuardNotForced | OpCode::GuardNotForced2 => {

majit/majit-backend/src/lib.rs

@@ -1433,8 +1433,15 @@ impl JitCellToken {
 
     /// Mark this loop as invalidated. Any subsequent execution of
     /// GUARD_NOT_INVALIDATED in the compiled code will fail.
+    /// `model.py:145 invalidate_loop`: activates the guards in the loop AND
+    /// all its attached bridges, so every bridge-generation flag minted so
+    /// far is set too. A bridge compiled after this call mints a fresh clear
+    /// flag and starts valid.
     pub fn invalidate(&self) {
         self.invalidated.store(true, Ordering::Release);
+        for flag in self.bridge_invalidation_flags.lock().iter() {
+            flag.store(true, Ordering::Release);
+        }
     }
 
     /// Load the compiled entry address written at backend `compile_loop`
@@ -3321,16 +3328,23 @@ mod tests {
     #[test]
     fn bridge_invalidation_flags_are_independent_generations() {
         let token = JitCellToken::new(42);
+        // A bridge attached before an invalidation is activated by it
+        // (model.py:145: "all GUARD_NOT_INVALIDATED in the loop and its
+        // attached bridges").
+        let pre_flag = token.mint_bridge_invalidation_flag();
         token.invalidate();
+        assert!(pre_flag.load(std::sync::atomic::Ordering::Acquire));
 
+        // A bridge compiled after the invalidation starts valid.
         let bridge_flag = token.mint_bridge_invalidation_flag();
         assert!(token.is_invalidated());
         assert!(!bridge_flag.load(std::sync::atomic::Ordering::Acquire));
 
         let flags = token.all_invalidation_flags();
-        assert_eq!(flags.len(), 2);
+        assert_eq!(flags.len(), 3);
         assert!(Arc::ptr_eq(&flags[0], &token.invalidation_flag()));
-        assert!(Arc::ptr_eq(&flags[1], &bridge_flag));
+        assert!(Arc::ptr_eq(&flags[1], &pre_flag));
+        assert!(Arc::ptr_eq(&flags[2], &bridge_flag));
         assert!(Arc::ptr_eq(
             &token.latest_bridge_invalidation_flag().unwrap(),
             &bridge_flag

majit/majit-gc/src/collector.rs

@@ -3460,6 +3460,12 @@ impl GcAllocator for MiniMarkGC {
     /// forced — old-gen is non-moving — so this is safe on unrooted
     /// host/interpreter paths. The end-of-major recompute corrects any drift.
     fn charge_oldgen_external(&mut self, obj_addr: usize, bytes: usize) {
+        // Nursery objects are the common case on this path and never old-gen;
+        // answer them with the O(1) range check instead of the old-gen
+        // membership probe (arena scan + rawmalloc hash lookup).
+        if self.nursery.contains(obj_addr) {
+            return;
+        }
         if self.oldgen.contains(obj_addr) {
             self.oldgen_external_bytes = self.oldgen_external_bytes.saturating_add(bytes);
         }

pyre/bench/synth/int_mul_ovf_bignum_promote.py

@@ -1,4 +1,4 @@
-# pyre-check: max-pypy-ratio=10
+# pyre-check: max-pypy-ratio=8
 
 # Overflow-crossing int multiply on a JIT-hot path. The inner loop is traced
 # while `scale` is small (a*a stays in machine-int range, so the recorded

pyre/pyre-interpreter/src/builtins.rs

@@ -4209,7 +4209,7 @@ fn builtin_issubclass(args: &[PyObjectRef]) -> Result<PyObjectRef, crate::PyErro
 
 // Descroperation helpers (lookup_type_special, try_dispatch_binary_special,
 // try_int_long_pow_with_modulo, binary_builtin_type_error,
-// box_bigint_result, issubtype_w) live in `crate::baseobjspace` because
+// issubtype_w) live in `crate::baseobjspace` because
 // they are space-level semantics shared between the builtin module,
 // weakproxy wrappers, and any future opcode dispatch.