diff --git a/majit/majit-backend-dynasm/src/aarch64/assembler.rs b/majit/majit-backend-dynasm/src/aarch64/assembler.rs index f79b4f5195d..b1d17d5a4bc 100644 --- a/majit/majit-backend-dynasm/src/aarch64/assembler.rs +++ b/majit/majit-backend-dynasm/src/aarch64/assembler.rs @@ -3490,7 +3490,20 @@ impl<'a> AssemblerARM64<'a> { self.emit_guard_no_exception_check(); self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs); } - OpCode::GuardNoOverflow | OpCode::GuardOverflow => { + OpCode::GuardNoOverflow => { + self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs); + } + OpCode::GuardOverflow => { + // aarch64/opassembler.py:547-551 aliases GUARD_NO_OVERFLOW to + // guard_true and GUARD_OVERFLOW to guard_false. The overflow + // arithmetic producer leaves the no-overflow success CC in + // `guard_success_cc`, so invert it for the expected-overflow + // arm before the common guard emitter derives its fail CC. + let no_overflow_cc = self + .guard_success_cc + .take() + .expect("GuardOverflow requires a preceding overflow operation"); + self.guard_success_cc = Some(invert_cc(no_overflow_cc)); self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs); } OpCode::GuardNotForced | OpCode::GuardNotForced2 => { diff --git a/majit/majit-backend-dynasm/src/j2plan.rs b/majit/majit-backend-dynasm/src/j2plan.rs index 334fb9b1378..71e19f6b755 100644 --- a/majit/majit-backend-dynasm/src/j2plan.rs +++ b/majit/majit-backend-dynasm/src/j2plan.rs @@ -165,24 +165,15 @@ pub(crate) struct TracePlan { pub inputargs: Vec, pub ops: Vec, pub live_points: Vec, - pub deopt_spill_points: Vec, pub max_live: usize, pub lowered_ops: usize, pub fallback_ops: usize, } -/// Guard fail args that are only needed on the deopt path at this point. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct DeoptSpillPoint { - pub op_index: usize, - pub args: Vec, -} - impl TracePlan { pub(crate) fn build(inputargs: &[InputArg], ops: &[Op]) -> Self { let lowered: Vec = ops.iter().map(|op| lower_op(op)).collect(); let live_points = compute_live_points(&lowered); - let deopt_spill_points = compute_deopt_spill_points(&lowered); let max_live = live_points .iter() .map(|point| point.live_in.len()) @@ -196,7 +187,6 @@ impl TracePlan { fallback_ops, ops: lowered, live_points, - deopt_spill_points, max_live, } } @@ -204,16 +194,6 @@ impl TracePlan { pub(crate) fn summary(&self) -> TracePlanSummary<'_> { TracePlanSummary(self) } - - pub(crate) fn deopt_spill_args_by_index(&self, len: usize) -> Vec> { - let mut by_index = vec![Vec::new(); len]; - for point in &self.deopt_spill_points { - if point.op_index < by_index.len() { - by_index[point.op_index] = point.args.clone(); - } - } - by_index - } } pub(crate) struct TracePlanSummary<'a>(&'a TracePlan); @@ -223,12 +203,11 @@ impl fmt::Display for TracePlanSummary<'_> { let plan = self.0; write!( f, - "ops={} lowered={} fallback={} max_live={} deopt_spills={}", + "ops={} lowered={} fallback={} max_live={}", plan.ops.len(), plan.lowered_ops, plan.fallback_ops, plan.max_live, - plan.deopt_spill_points.len() ) } } @@ -400,33 +379,6 @@ fn compute_live_points(ops: &[LirOp]) -> Vec { points } -fn compute_deopt_spill_points(ops: &[LirOp]) -> Vec { - let mut fast_live_after = Vec::new(); - let mut points = Vec::new(); - - for (op_index, op) in ops.iter().enumerate().rev() { - if let LirOp::Guard { fail_args, .. } = op { - let mut args = Vec::new(); - for &arg in fail_args { - if !fast_live_after.contains(&arg) { - add_ref(&mut args, arg); - } - } - if !args.is_empty() { - points.push(DeoptSpillPoint { op_index, args }); - } - } - - if let Some(dst) = op.def() { - remove_ref(&mut fast_live_after, dst); - } - op.add_uses(&mut fast_live_after); - } - - points.reverse(); - points -} - fn int_bin_kind(opcode: OpCode) -> IntBinKind { match opcode { OpCode::IntAdd => IntBinKind::Add, @@ -665,7 +617,6 @@ mod tests { ); assert_eq!(plan.fallback_ops, 0); - assert!(plan.deopt_spill_points.is_empty()); assert!(matches!( plan.ops[1], LirOp::IntBin { @@ -715,44 +666,11 @@ mod tests { assert!(guard_live.contains(&OpRef::int_op(1))); assert!(guard_live.contains(&OpRef::int_op(2))); - assert_eq!( - plan.deopt_spill_points, - vec![super::DeoptSpillPoint { - op_index: 2, - args: vec![OpRef::int_op(1)] - }] - ); - let add_live = &plan.live_points[0].live_in; assert!(add_live.contains(&i0)); assert!(!add_live.contains(&c1)); } - #[test] - fn deopt_spill_point_keeps_jump_args_on_fast_path() { - let i0 = OpRef::int_op(0); - let c1 = OpRef::const_int(1); - - let add = Op::new(OpCode::IntAdd, &[rb(i0), rb(c1)]); - add.pos.set(OpRef::int_op(1)); - - let is_true = Op::new(OpCode::IntIsTrue, &[rb(OpRef::int_op(1))]); - is_true.pos.set(OpRef::int_op(2)); - - let guard = Op::new(OpCode::GuardTrue, &[rb(OpRef::int_op(2))]); - guard.pos.set(OpRef::int_op(3)); - guard.setfailargs(vec![rb(OpRef::int_op(1))].into()); - let jump = Op::new(OpCode::Jump, &[rb(OpRef::int_op(1))]); - jump.pos.set(OpRef::int_op(4)); - - let plan = TracePlan::build( - &[InputArg::from_type(Type::Int, 0)], - &[add, is_true, guard, jump], - ); - - assert!(plan.deopt_spill_points.is_empty()); - } - #[test] fn lowers_indexed_memory_operands_by_role() { let base = OpRef::int_op(0); diff --git a/majit/majit-backend-dynasm/src/regalloc.rs b/majit/majit-backend-dynasm/src/regalloc.rs index 511c7df0460..3fa557e36a0 100644 --- a/majit/majit-backend-dynasm/src/regalloc.rs +++ b/majit/majit-backend-dynasm/src/regalloc.rs @@ -1692,10 +1692,6 @@ pub struct RegAlloc<'a> { /// closing JUMP, used by `_compute_hint_locations_from_descr` to pin /// reg hints via `longevity.fixed_register(position, reg, box)`. final_jump_op_position: i32, - /// j2-style deopt-only fail args, indexed by original operation index. - /// These values are needed by guard recovery but not by the fast path - /// after the guard, so keeping them in registers only increases pressure. - j2_deopt_spill_args: Vec>, /// j2-lowered operations, indexed by original operation index. The main /// dispatch path consumes these; legacy opcode dispatch is a guard rail /// only if a plan entry is missing. @@ -1736,7 +1732,6 @@ impl<'a> RegAlloc<'a> { jump_target_descr: None, final_jump_args: None, final_jump_op_position: -1, - j2_deopt_spill_args: Vec::new(), j2_ops: Vec::new(), temp_var_counter: 0, } @@ -1829,7 +1824,6 @@ impl<'a> RegAlloc<'a> { self.final_jump_args = None; self.final_jump_op_position = -1; let j2_plan = crate::j2plan::TracePlan::build(self.inputargs, self.operations); - self.j2_deopt_spill_args = j2_plan.deopt_spill_args_by_index(self.operations.len()); self.j2_ops = j2_plan.ops; // x86/regalloc.py:191 X86RegisterHints().add_hints(longevity, inputargs, operations) // @@ -2246,7 +2240,6 @@ impl<'a> RegAlloc<'a> { result_loc: Option, output: &mut Vec, ) { - self.spill_j2_deopt_args(op_index); self.flush_moves(output); let faillocs = self.locs_for_fail(op); output.push(RegAllocOp::PerformGuard { @@ -2265,7 +2258,6 @@ impl<'a> RegAlloc<'a> { result_loc: Option, output: &mut Vec, ) { - self.spill_j2_deopt_args(op_index); self.flush_moves(output); let faillocs = self.locs_for_fail_args(fail_args); output.push(RegAllocOp::PerformGuard { @@ -2276,56 +2268,6 @@ impl<'a> RegAlloc<'a> { }); } - fn spill_j2_deopt_args(&mut self, op_index: usize) { - let Some(args) = self.j2_deopt_spill_args.get(op_index).cloned() else { - return; - }; - for arg in args { - if arg.is_none() || arg.is_constant() { - continue; - } - let tp = self.tp(arg); - if tp == Type::Float { - Self::spill_deopt_arg_from_manager( - &mut self.xrm, - arg, - tp, - &mut self.longevity, - &mut self.fm, - ); - } else { - Self::spill_deopt_arg_from_manager( - &mut self.rm, - arg, - tp, - &mut self.longevity, - &mut self.fm, - ); - } - } - } - - fn spill_deopt_arg_from_manager( - mgr: &mut RegisterManager, - arg: OpRef, - tp: Type, - longevity: &mut LifetimeManager, - fm: &mut FrameManager, - ) { - let Some(reg) = mgr.reg_bindings_get(arg, longevity) else { - return; - }; - mgr._sync_var_to_stack(arg, tp, longevity, fm); - mgr.reg_bindings_del(arg, longevity); - mgr.free_regs.push(reg); - if crate::majit_log_enabled() { - eprintln!( - "[dynasm:j2plan] spill deopt-only failarg {:?} from {:?}", - arg, reg - ); - } - } - /// aarch64/regalloc.py:1089 get_gcmap. pub fn get_gcmap(&self, forbidden_regs: &[RegLoc], noregs: bool) -> *mut usize { let frame_depth = self.fm.get_frame_depth(); @@ -6321,7 +6263,13 @@ mod tests { } #[test] - fn test_j2_deopt_only_failarg_spilled_before_guard() { + fn test_j2_deopt_only_failarg_kept_in_register_at_guard() { + // A deopt-only fail arg (dead on the fast path after the guard) is + // captured from its register at the guard, not eagerly spilled to a + // frame slot: the failure path saves every register before rebuilding + // the frame, so a register faillocs is fully recoverable, and the + // fast path pays no spill store. + // // i0 is the typed inputarg slot 0 (Int) that the regalloc // registers from `inputargs.opref()`. i1/i2 are op result // positions; their variant tag is unconstrained, so plain @@ -6368,8 +6316,8 @@ mod tests { panic!("guard op was not lowered through RegAllocOp::PerformGuard"); }; assert!( - matches!(faillocs.as_slice(), [Some(Loc::Frame(_))]), - "deopt-only failarg should be captured from a frame slot: {:?}", + matches!(faillocs.as_slice(), [Some(Loc::Reg(_))]), + "deopt-only failarg should be captured from a register: {:?}", faillocs ); } diff --git a/majit/majit-backend-dynasm/src/x86/assembler.rs b/majit/majit-backend-dynasm/src/x86/assembler.rs index 60c6c288878..74446345037 100644 --- a/majit/majit-backend-dynasm/src/x86/assembler.rs +++ b/majit/majit-backend-dynasm/src/x86/assembler.rs @@ -4651,7 +4651,20 @@ impl<'a> Assembler386<'a> { self.emit_guard_no_exception_check(); self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs); } - OpCode::GuardNoOverflow | OpCode::GuardOverflow => { + OpCode::GuardNoOverflow => { + self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs); + } + OpCode::GuardOverflow => { + // x86/assembler.py:1873-1874 aliases GUARD_NO_OVERFLOW to + // guard_true and GUARD_OVERFLOW to guard_false. The overflow + // arithmetic producer leaves the no-overflow success CC in + // `guard_success_cc`, so invert it for the expected-overflow + // arm before the common guard emitter derives its fail CC. + let no_overflow_cc = self + .guard_success_cc + .take() + .expect("GuardOverflow requires a preceding overflow operation"); + self.guard_success_cc = Some(invert_cc(no_overflow_cc)); self.implement_guard_with_faillocs(op, op_index, fail_index, faillocs); } OpCode::GuardNotForced | OpCode::GuardNotForced2 => { diff --git a/majit/majit-backend/src/lib.rs b/majit/majit-backend/src/lib.rs index 23836f05711..fd593708c13 100644 --- a/majit/majit-backend/src/lib.rs +++ b/majit/majit-backend/src/lib.rs @@ -1433,8 +1433,15 @@ impl JitCellToken { /// Mark this loop as invalidated. Any subsequent execution of /// GUARD_NOT_INVALIDATED in the compiled code will fail. + /// `model.py:145 invalidate_loop`: activates the guards in the loop AND + /// all its attached bridges, so every bridge-generation flag minted so + /// far is set too. A bridge compiled after this call mints a fresh clear + /// flag and starts valid. pub fn invalidate(&self) { self.invalidated.store(true, Ordering::Release); + for flag in self.bridge_invalidation_flags.lock().iter() { + flag.store(true, Ordering::Release); + } } /// Load the compiled entry address written at backend `compile_loop` @@ -3321,16 +3328,23 @@ mod tests { #[test] fn bridge_invalidation_flags_are_independent_generations() { let token = JitCellToken::new(42); + // A bridge attached before an invalidation is activated by it + // (model.py:145: "all GUARD_NOT_INVALIDATED in the loop and its + // attached bridges"). + let pre_flag = token.mint_bridge_invalidation_flag(); token.invalidate(); + assert!(pre_flag.load(std::sync::atomic::Ordering::Acquire)); + // A bridge compiled after the invalidation starts valid. let bridge_flag = token.mint_bridge_invalidation_flag(); assert!(token.is_invalidated()); assert!(!bridge_flag.load(std::sync::atomic::Ordering::Acquire)); let flags = token.all_invalidation_flags(); - assert_eq!(flags.len(), 2); + assert_eq!(flags.len(), 3); assert!(Arc::ptr_eq(&flags[0], &token.invalidation_flag())); - assert!(Arc::ptr_eq(&flags[1], &bridge_flag)); + assert!(Arc::ptr_eq(&flags[1], &pre_flag)); + assert!(Arc::ptr_eq(&flags[2], &bridge_flag)); assert!(Arc::ptr_eq( &token.latest_bridge_invalidation_flag().unwrap(), &bridge_flag diff --git a/majit/majit-gc/src/collector.rs b/majit/majit-gc/src/collector.rs index 76920a0dc62..5df9d6e6359 100644 --- a/majit/majit-gc/src/collector.rs +++ b/majit/majit-gc/src/collector.rs @@ -3460,6 +3460,12 @@ impl GcAllocator for MiniMarkGC { /// forced — old-gen is non-moving — so this is safe on unrooted /// host/interpreter paths. The end-of-major recompute corrects any drift. fn charge_oldgen_external(&mut self, obj_addr: usize, bytes: usize) { + // Nursery objects are the common case on this path and never old-gen; + // answer them with the O(1) range check instead of the old-gen + // membership probe (arena scan + rawmalloc hash lookup). + if self.nursery.contains(obj_addr) { + return; + } if self.oldgen.contains(obj_addr) { self.oldgen_external_bytes = self.oldgen_external_bytes.saturating_add(bytes); } diff --git a/pyre/bench/synth/int_mul_ovf_bignum_promote.py b/pyre/bench/synth/int_mul_ovf_bignum_promote.py index eaa3cee58b8..2ced4a4dcba 100644 --- a/pyre/bench/synth/int_mul_ovf_bignum_promote.py +++ b/pyre/bench/synth/int_mul_ovf_bignum_promote.py @@ -1,4 +1,4 @@ -# pyre-check: max-pypy-ratio=10 +# pyre-check: max-pypy-ratio=8 # Overflow-crossing int multiply on a JIT-hot path. The inner loop is traced # while `scale` is small (a*a stays in machine-int range, so the recorded diff --git a/pyre/pyre-interpreter/src/builtins.rs b/pyre/pyre-interpreter/src/builtins.rs index 5a54ec20dc3..0a014a4f7bc 100644 --- a/pyre/pyre-interpreter/src/builtins.rs +++ b/pyre/pyre-interpreter/src/builtins.rs @@ -4209,7 +4209,7 @@ fn builtin_issubclass(args: &[PyObjectRef]) -> Result PyObjectRef { - crate::objspace::descroperation::box_bigint_result(self.pack()) + crate::objspace::descroperation::bigint_result(self.pack()) } } @@ -1265,9 +1265,9 @@ impl W_TextIOWrapper { let position_cookie = PositionCookie::unpack(position)?; super::call_method_result(self.self_obj(), "flush", &[])?; - let start = crate::objspace::descroperation::box_bigint_result( - malachite_bigint::BigInt::from(position_cookie.start_pos), - ); + let start = crate::objspace::descroperation::bigint_result(malachite_bigint::BigInt::from( + position_cookie.start_pos, + )); self.call_buffer("seek", &[start])?; self.decoded.reset(); self.snapshot = None; diff --git a/pyre/pyre-interpreter/src/objspace/descroperation.rs b/pyre/pyre-interpreter/src/objspace/descroperation.rs index 950d17a36f1..d4cef19be2b 100644 --- a/pyre/pyre-interpreter/src/objspace/descroperation.rs +++ b/pyre/pyre-interpreter/src/objspace/descroperation.rs @@ -39,7 +39,7 @@ unsafe fn as_bigint(obj: PyObjectRef) -> BigInt { /// Box a BigInt result, demoting to W_IntObject if it fits in i64. -fn bigint_result(value: BigInt) -> PyObjectRef { +pub(crate) fn bigint_result(value: BigInt) -> PyObjectRef { if jit_bigint_to_i64_fits(&value) != 0 { w_int_new(jit_bigint_to_i64_value(&value)) } else { @@ -489,15 +489,15 @@ unsafe fn int_mod(a: PyObjectRef, b: PyObjectRef) -> PyResult { // ── Long (BigInt) arithmetic operations ───────────────────────────── unsafe fn long_add(a: PyObjectRef, b: PyObjectRef) -> PyResult { - Ok(bigint_result(bigint_add(as_bigint(a), as_bigint(b)))) + Ok(w_long_new(bigint_add(as_bigint(a), as_bigint(b)))) } unsafe fn long_sub(a: PyObjectRef, b: PyObjectRef) -> PyResult { - Ok(bigint_result(bigint_sub(as_bigint(a), as_bigint(b)))) + Ok(w_long_new(bigint_sub(as_bigint(a), as_bigint(b)))) } unsafe fn long_mul(a: PyObjectRef, b: PyObjectRef) -> PyResult { - Ok(bigint_result(bigint_mul(as_bigint(a), as_bigint(b)))) + Ok(w_long_new(bigint_mul(as_bigint(a), as_bigint(b)))) } unsafe fn long_floordiv(a: PyObjectRef, b: PyObjectRef) -> PyResult { @@ -509,7 +509,8 @@ unsafe fn long_floordiv(a: PyObjectRef, b: PyObjectRef) -> PyResult { return Err(PyError::zero_division("integer division or modulo by zero")); } // rbigint.floordiv → _divmod, returning the quotient half (rbigint.py:1001). - Ok(bigint_result(as_bigint(a).div_mod_floor(&vb).0)) + // `_floordiv`/`_int_floordiv` both `newlong` the quotient, keeping a long. + Ok(w_long_new(as_bigint(a).div_mod_floor(&vb).0)) } unsafe fn long_mod(a: PyObjectRef, b: PyObjectRef) -> PyResult { @@ -520,7 +521,15 @@ unsafe fn long_mod(a: PyObjectRef, b: PyObjectRef) -> PyResult { return Err(PyError::zero_division("integer modulo by zero")); } // rbigint.mod → _divmod, returning the remainder half (rbigint.py:1001). - Ok(bigint_result(as_bigint(a).div_mod_floor(&vb).1)) + // `_int_mod` (machine-int RHS) returns `space.newint` — the remainder of a + // long by a machine int always fits — while `_mod` (long RHS) returns + // `newlong`. `a` is a long here, so the RHS kind decides. + let r = as_bigint(a).div_mod_floor(&vb).1; + if is_int_like(b) { + Ok(w_int_new(jit_bigint_to_i64_value(&r))) + } else { + Ok(w_long_new(r)) + } } /// `rbigint.floordiv` payload half (`longobject.py:409 _floordiv` → @@ -925,29 +934,31 @@ unsafe fn long_pow(a: PyObjectRef, b: PyObjectRef) -> PyResult { let fb = as_float(b); return Ok(w_float_new(float_pow_raw(fa, fb)?)); } - // longobject.py:229: `if not exp_bigint: return int_pow(0)` → 1. + // longobject.py:229: `if not exp_bigint: return int_pow(0)` → 1. `descr_pow` + // wraps every branch as `W_LongObject`, so a long base keeps the long + // representation across these trivial-base short-circuits too. if pyre_object::longobject::jit_bigint_sign_i64(&vb) == 0 { - return Ok(w_int_new(1)); + return Ok(w_long_new(BigInt::from(1))); } // longobject.py:224-231: rbigint.pow handles arbitrary exponents. // Short-circuit trivial bases before the u32 narrowing so that // 1 ** huge, (-1) ** huge, 0 ** huge succeed. let va = as_bigint(a); if pyre_object::longobject::jit_bigint_sign_i64(&va) == 0 { - return Ok(w_int_new(0)); + return Ok(w_long_new(BigInt::from(0))); } if va == BigInt::from(1) { - return Ok(w_int_new(1)); + return Ok(w_long_new(BigInt::from(1))); } if va == BigInt::from(-1) { let even = vb.clone() % BigInt::from(2) == BigInt::from(0); - return Ok(w_int_new(if even { 1 } else { -1 })); + return Ok(w_long_new(BigInt::from(if even { 1 } else { -1 }))); } let exp = match vb.to_u32() { Some(v) => v, None => return Err(PyError::memory_error("exponent too large")), }; - Ok(bigint_result(va.pow(exp))) + Ok(w_long_new(va.pow(exp))) } // ── Shift operations ───────────────────────────────────────────────── @@ -998,11 +1009,12 @@ unsafe fn long_lshift(a: PyObjectRef, b: PyObjectRef) -> PyResult { } else { let va = as_bigint(a); if pyre_object::longobject::jit_bigint_sign_i64(&va) == 0 { - return Ok(w_int_new(0)); + // `_lshift` returns `self` (a W_LongObject) for a zero base. + return Ok(w_long_new(BigInt::from(0))); } return Err(PyError::overflow_error("shift count too large")); }; - Ok(bigint_result(checked_bigint_lshift(as_bigint(a), shift)?)) + Ok(w_long_new(checked_bigint_lshift(as_bigint(a), shift)?)) } unsafe fn long_rshift(a: PyObjectRef, b: PyObjectRef) -> PyResult { @@ -1024,7 +1036,8 @@ unsafe fn long_rshift(a: PyObjectRef, b: PyObjectRef) -> PyResult { }, )); }; - Ok(bigint_result(bigint_rshift(as_bigint(a), shift))) + // `_rshift`/`_int_rshift` `newlong` the normal-count result, keeping a long. + Ok(w_long_new(bigint_rshift(as_bigint(a), shift))) } // ── bool-as-int helpers ────────────────────────────────────────────── @@ -1062,15 +1075,15 @@ unsafe fn int_bitxor(a: PyObjectRef, b: PyObjectRef) -> PyResult { } unsafe fn long_bitand(a: PyObjectRef, b: PyObjectRef) -> PyResult { - Ok(bigint_result(bigint_and(as_bigint(a), as_bigint(b)))) + Ok(w_long_new(bigint_and(as_bigint(a), as_bigint(b)))) } unsafe fn long_bitor(a: PyObjectRef, b: PyObjectRef) -> PyResult { - Ok(bigint_result(bigint_or(as_bigint(a), as_bigint(b)))) + Ok(w_long_new(bigint_or(as_bigint(a), as_bigint(b)))) } unsafe fn long_bitxor(a: PyObjectRef, b: PyObjectRef) -> PyResult { - Ok(bigint_result(bigint_xor(as_bigint(a), as_bigint(b)))) + Ok(w_long_new(bigint_xor(as_bigint(a), as_bigint(b)))) } // ── String operations ──────────────────────────────────────────────── @@ -2627,6 +2640,14 @@ pub(crate) fn divmod_builtin(a: PyObjectRef, b: PyObjectRef) -> PyResult { if !is_true(b)? { return Err(PyError::zero_division("division by zero")); } + // `_divmod`/`_int_divmod` both `newtuple2(newlong(div), newlong(mod))` + // — a long receiver keeps BOTH parts as W_LongObject, so bypass the + // remainder demote that `mod_` applies for an int RHS. An int + // receiver (or float) still routes through floordiv/mod. + if is_long(a) && is_int_or_long(b) { + let (q, r) = as_bigint(a).div_mod_floor(&as_bigint(b)); + return Ok(w_tuple_new(vec![w_long_new(q), w_long_new(r)])); + } let q = floordiv(a, b)?; let r = mod_(a, b)?; return Ok(w_tuple_new(vec![q, r])); @@ -2859,6 +2880,13 @@ pub(crate) fn try_int_long_pow_with_modulo( return Ok(None); } + // `descr_pow` wraps its 3-arg result as `W_LongObject` whenever the + // receiver is a long; the int path (`W_IntObject.descr_pow`) reaches + // that same long path via `_pow_ovf2long` when the exponent or modulus + // is a long. So the result stays a long unless all three operands are + // machine ints, in which case `space.newint` demotes it. + let all_int_like = is_int_like(base) && is_int_like(exp) && is_int_like(modulus); + let base = crate::builtins::obj_to_bigint(base); let exp = crate::builtins::obj_to_bigint(exp); let modulus = crate::builtins::obj_to_bigint(modulus); @@ -2890,13 +2918,15 @@ pub(crate) fn try_int_long_pow_with_modulo( if negative_modulus && bigint_gt(result.clone(), BigInt::from(0)) { result = bigint_sub(result, abs_modulus); } - return Ok(Some(box_bigint_result(result))); + return Ok(Some(pow_mod_result(result, all_int_like))); } if bigint_eq(exp.clone(), BigInt::from(0)) { - return Ok(Some(box_bigint_result(bigint_mod( - BigInt::from(1), - modulus, - )))); + // `x ** 0 % m` is `1 % m` under floor semantics, so a negative + // modulus yields a negative residue (`pow(2, 0, -13) == -12`). + return Ok(Some(pow_mod_result( + BigInt::from(1).mod_floor(&modulus), + all_int_like, + ))); } let negative_modulus = bigint_lt(modulus.clone(), BigInt::from(0)); @@ -2909,12 +2939,15 @@ pub(crate) fn try_int_long_pow_with_modulo( if negative_modulus && bigint_gt(result.clone(), BigInt::from(0)) { result = bigint_sub(result, abs_modulus); } - Ok(Some(box_bigint_result(result))) + Ok(Some(pow_mod_result(result, all_int_like))) } } -pub(crate) fn box_bigint_result(value: BigInt) -> PyObjectRef { - if jit_bigint_to_i64_fits(&value) != 0 { +/// Box a 3-arg `pow` result: `space.newint` (demote) when every operand was a +/// machine int, else `W_LongObject` (a long receiver keeps the long +/// representation). The demote arm always fits — `result < |modulus|`. +fn pow_mod_result(value: BigInt, all_int_like: bool) -> PyObjectRef { + if all_int_like { w_int_new(jit_bigint_to_i64_value(&value)) } else { w_long_new(value) @@ -2998,6 +3031,12 @@ pub fn divmod(a: PyObjectRef, b: PyObjectRef) -> PyResult { if !is_true(b)? { return Err(PyError::zero_division("division by zero")); } + // A long receiver keeps both divmod parts as W_LongObject (see + // `divmod_builtin`); bypass the remainder demote for an int RHS. + if is_long(a) && is_int_or_long(b) { + let (q, r) = as_bigint(a).div_mod_floor(&as_bigint(b)); + return Ok(w_tuple_new(vec![w_long_new(q), w_long_new(r)])); + } let q = floordiv(a, b)?; let r = mod_(a, b)?; return Ok(w_tuple_new(vec![q, r])); @@ -3753,7 +3792,7 @@ pub fn pos(a: PyObjectRef) -> PyResult { return Ok(w_int_new(int_value(a))); } if is_long(a) { - return Ok(bigint_result(w_long_get_value(a).clone())); + return Ok(w_long_new(w_long_get_value(a).clone())); } if is_float(a) { return Ok(w_float_new(w_float_get_value(a))); @@ -3792,7 +3831,7 @@ pub fn neg(a: PyObjectRef) -> PyResult { }; } if is_long(a) { - return Ok(bigint_result(bigint_neg(w_long_get_value(a).clone()))); + return Ok(w_long_new(bigint_neg(w_long_get_value(a).clone()))); } if is_float(a) { return Ok(w_float_new(-w_float_get_value(a))); @@ -3840,7 +3879,7 @@ the bitwise inversion of the underlying int.", return Ok(w_int_new(!int_value(a))); } if is_long(a) { - return Ok(bigint_result(bigint_invert(w_long_get_value(a).clone()))); + return Ok(w_long_new(bigint_invert(w_long_get_value(a).clone()))); } if let Some(result) = try_instance_unaryop(a, "__invert__")? { return Ok(result); @@ -3945,14 +3984,16 @@ mod tests { } #[test] - fn test_long_demote_to_int() { - // long + long that fits back in i64 → W_IntObject + fn test_long_add_keeps_long_when_fits() { + // long + int whose sum fits back in i64 stays a W_LongObject: `newlong` + // never demotes (withsmalllong=False), so a shrunk long keeps the long + // representation. let a = w_long_new(BigInt::from(i64::MAX) + BigInt::from(1)); let b = w_int_new(-1); let result = add(a, b).unwrap(); unsafe { - assert!(is_int(result)); - assert_eq!(w_int_get_value(result), i64::MAX); + assert!(is_long(result)); + assert_eq!(*w_long_get_value(result), BigInt::from(i64::MAX)); } } @@ -4163,7 +4204,11 @@ mod tests { let a = w_long_new(BigInt::from(i64::MAX) + BigInt::from(1)); let b = w_int_new(0xFF); let result = and_(a, b).unwrap(); - unsafe { assert_eq!(w_int_get_value(result), 0) }; + // long & int keeps a W_LongObject even when the result fits (newlong). + unsafe { + assert!(is_long(result)); + assert_eq!(*w_long_get_value(result), BigInt::from(0)); + } } #[test] diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index 1e91e3000ac..6a611d7ede8 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -570,7 +570,7 @@ pub(crate) fn recipe_parent_frame_from_recipe( .find(|op| op.next_pc == entry && op.opname.starts_with("residual_call")) .map(|op| op.pc); let resume_marker_jit_pc = - call_jit_pc.and_then(|pc| pjc.after_residual_marker_for_jitcode_pc(pc)); + call_jit_pc.and_then(|pc| super::resume_snapshot::inline_call_return_marker(&pjc, pc)); // Reconstruct this paused parent frame's vable + ec (the same // `emit_new_pyframe_inline_with_params` the deepest-callee setup uses) so diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 241c60f4772..02196e5a4cc 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -6064,6 +6064,70 @@ fn walker_unbox_int_typed( )) } +/// Walker-native unbox of a fits-int `W_LongObject` operand into a raw i64 — +/// the long analogue of [`walker_unbox_int`], mirroring +/// [`crate::state::trace_unbox_long_with_resume`] but emitting the walker +/// snapshot guards. `is_plain_int1` accepts a `W_LongObject` whose BigInt +/// fits a machine word (`type(w_obj) is W_LongObject and w_obj._fits_int()`, +/// listobject.py), and `plain_int_w` unwraps it via `_int_w`: +/// +/// 1. `GUARD_CLASS(obj, LONG_TYPE)` when the class is not already known. +/// 2. `residual_call(jit_w_long_fits_int, obj)` + `GUARD_TRUE` — a later +/// execution may see the BigInt grown out of i64 range, so re-probe and +/// deopt to the residual on a non-fitting arrival. +/// 3. `residual_call(jit_w_long_toint, obj)` — `rbigint.toint()`, statically +/// non-raising after the fits guard. +/// +/// A `W_LongObject` is a distinct 8-aligned heap struct and is never a tagged +/// immediate, so the `GUARD_CLASS` deref needs no lowbit pre-test. +fn walker_unbox_long( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + obj: OpRef, + long_type_addr: i64, +) -> Result { + if !ctx.trace_ctx.heap_cache().is_class_known(obj) { + let type_const = ctx.trace_ctx.const_int(long_type_addr); + ctx.trace_ctx + .record_guard(OpCode::GuardClass, &[obj, type_const], 0); + walker_capture_snapshot_for_last_guard(ctx, op_pc)?; + ctx.trace_ctx + .heap_cache_mut() + .class_now_known(obj, long_type_addr); + } + let obj_concrete = walker_concrete_ref_object(ctx, obj); + let fits_fn = pyre_object::longobject::jit_w_long_fits_int as *const (); + let fits = ctx.trace_ctx.call_typed_with_effect( + OpCode::CallI, + fits_fn, + &[obj], + &[majit_ir::Type::Ref], + majit_ir::Type::Int, + majit_metainterp::cannot_raise_effect_info(), + ); + if let Some(o) = obj_concrete { + let fits_concrete = pyre_object::longobject::jit_w_long_fits_int(o as usize as i64); + ctx.trace_ctx + .set_opref_concrete(fits, majit_ir::Value::Int(fits_concrete)); + } + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[fits])?; + let toint_fn = pyre_object::longobject::jit_w_long_toint as *const (); + let raw = ctx.trace_ctx.call_typed_with_effect( + OpCode::CallI, + toint_fn, + &[obj], + &[majit_ir::Type::Ref], + majit_ir::Type::Int, + majit_metainterp::cannot_raise_effect_info(), + ); + if let Some(o) = obj_concrete { + let v = pyre_object::longobject::jit_w_long_toint(o as usize as i64); + ctx.trace_ctx + .set_opref_concrete(raw, majit_ir::Value::Int(v)); + } + Ok(raw) +} + /// Walker-native unbox of a boxed `W_FloatObject` operand: the float /// analogue of [`walker_unbox_int`]. `GUARD_CLASS` (with the walker /// snapshot) when the operand's class is not yet known, then the ctx-only diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index 89b7c2ed22b..8d9d4d7cfc8 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -1317,6 +1317,26 @@ fn capture_inline_parent_blackhole( }) } +/// Return the exact JitCode coordinate at which a paused caller continues +/// after an inlined residual call. +/// +/// RPython's `MIFrame.pc` remains immediately after the call instruction +/// while the callee runs. Preserve that shape here: blackhole return setup +/// walks backward from this coordinate to recover the call's destination +/// register. Advancing to a later semantic-fallthrough marker skips the +/// intervening `live` / virtualizable synchronization instructions and makes +/// that backward walk read an unrelated operand byte. +pub(crate) fn inline_call_return_marker( + pjc: &crate::pyjitcode::PyJitCode, + call_jit_pc: usize, +) -> Option { + let marker = + crate::jitcode_runtime::decode_op_at(pjc.jitcode.code.as_slice(), call_jit_pc)?.next_pc; + pjc.jitcode + .can_decode_live_vars(marker, crate::state::op_live()) + .then_some(marker) +} + pub(crate) fn compute_inline_caller_frame( ctx: &mut WalkContext<'_, '_, Sym>, call_jit_pc: usize, @@ -1383,7 +1403,7 @@ pub(crate) fn compute_inline_caller_frame( ( jc.index as u32, fallthrough, - jc.payload.after_residual_marker_for_jitcode_pc(call_jit_pc), + inline_call_return_marker(&jc.payload, call_jit_pc), jc.payload.code_ptr, ) }; @@ -1450,8 +1470,8 @@ pub(crate) fn compute_inline_caller_frame( if result_color < ctx.registers_r.len() { ctx.registers_r[result_color] = null_ref; } - // The after-residual marker names the same `-live-` the fallthrough - // translation resolves to (M73_PFMARKER identity), bypassing the py channel. + // Keep the caller at the immediate post-call `-live-`, matching the + // paused `MIFrame.pc` used by RPython and the blackhole return ABI. let caller_liveness_word = match resume_marker_jit_pc { Some(m) => m as i32, None => majit_ir::resumedata::NO_JITCODE_PC, @@ -1504,7 +1524,7 @@ pub(crate) fn compute_nested_inline_caller_frame( if !pjc.is_populated() || pjc.code_ptr.is_null() { return Err(InlineCallerFrameDecline::Unavailable); } - let resume_marker_jit_pc = pjc.after_residual_marker_for_jitcode_pc(call_jit_pc); + let resume_marker_jit_pc = inline_call_return_marker(&pjc, call_jit_pc); let after_residual_call_resume = pjc.after_residual_call_resume_for_jitcode_pc(call_jit_pc); // A CALL inside a try-block at inline depth ≥2: the rejoin-loop lift is // scoped to the top-level caller (`compute_inline_caller_frame`) for now, so @@ -1572,8 +1592,8 @@ pub(crate) fn compute_nested_inline_caller_frame( if result_color < ctx.registers_r.len() { ctx.registers_r[result_color] = null_ref; } - // The after-residual marker names the same `-live-` the fallthrough - // translation resolves to (M73_PFMARKER identity), bypassing the py channel. + // Keep the caller at the immediate post-call `-live-`, matching the + // paused `MIFrame.pc` used by RPython and the blackhole return ABI. // Without a marker there is no coordinate to encode against, so the // sentinel declines the caller frame. let caller_liveness_word = match resume_marker_jit_pc { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 2445ece03cb..b980fa83af5 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -417,24 +417,6 @@ pub(crate) fn try_walker_specialize_binary_op_int( return Ok(None); }; - // intobject.py `_make_generic_descr_binop`: the machine-int fast path - // exits through `_make_ovf2long` when `ovfcheck(op(x, y))` raises. - // This walker specialization represents only the no-overflow arm. When - // the concrete walk is already on the overflow arm, defer before emitting - // any IR so the generic residual records the authentic W_Long result. - // Emitting INT_*_OVF + GUARD_NO_OVERFLOW here would contradict that - // result: the bridge would immediately try to prove the freshly boxed - // W_IntObject is a W_LongObject and abort on every retry. - let overflowed = match op_code { - OpCode::IntAddOvf => la.checked_add(rb).is_none(), - OpCode::IntSubOvf => la.checked_sub(rb).is_none(), - OpCode::IntMulOvf => la.checked_mul(rb).is_none(), - _ => false, - }; - if overflowed { - return Ok(None); - } - // intobject.py range validation (mirror the former int fast path's // needs_concrete_check): bail to the generic leg when the bare-IR-op // emission would be unsound (zero / INT_MIN-overflow divisor, oversized @@ -478,6 +460,28 @@ pub(crate) fn try_walker_specialize_binary_op_int( } } + // pyjitpl.py:1881 handle_possible_overflow_error follows the concrete + // Add/Sub/Mul outcome. The overflowing arm mirrors intobject.py:494 + // _make_ovf2long: guard_overflow, call the elidable raw-int bigint helper + // (rbigint.py:717/788/873), guard the newlong demote attempt, and inline + // the W_LongObject box instead of falling through to the generic + // CallMayForceR BINARY_OP leg. + let overflows = has_overflow + && match op_code { + OpCode::IntAddOvf => la.checked_add(rb).is_none(), + OpCode::IntSubOvf => la.checked_sub(rb).is_none(), + OpCode::IntMulOvf => la.checked_mul(rb).is_none(), + _ => false, + }; + if overflows { + let boxed_result_obj = boxed_result_i64 as usize as pyre_object::PyObjectRef; + if boxed_result_obj == pyre_object::PY_NULL + || !unsafe { pyre_object::is_long(boxed_result_obj) } + { + return Ok(None); + } + } + // --- emit the specialized IR (walker-native) --- // bool and int share `intval`; guard each operand against its own vtable // (BOOL_TYPE / INT_TYPE) so a bool unboxes through its own class. @@ -485,6 +489,78 @@ pub(crate) fn try_walker_specialize_binary_op_int( let (rhs_type, rhs_descr) = crate::state::int_or_bool_unbox_type_descr(rhs_obj); let lhs_raw = walker_unbox_int_typed(ctx, op_pc, lhs, lhs_type, lhs_descr)?; let rhs_raw = walker_unbox_int_typed(ctx, op_pc, rhs, rhs_type, rhs_descr)?; + if overflows { + let boxed_result_obj = boxed_result_i64 as usize as pyre_object::PyObjectRef; + let concrete_value = match op_code { + OpCode::IntAddOvf => la.wrapping_add(rb), + OpCode::IntSubOvf => la.wrapping_sub(rb), + OpCode::IntMulOvf => la.wrapping_mul(rb), + _ => unreachable!("overflow arm requires Add/Sub/Mul"), + }; + let raw_result = ctx.trace_ctx.record_op(op_code, &[lhs_raw, rhs_raw]); + ctx.trace_ctx + .set_opref_concrete(raw_result, majit_ir::Value::Int(concrete_value)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardOverflow, &[])?; + + let payload_concrete = unsafe { + *((boxed_result_obj as *const u8).add(pyre_object::longobject::LONG_VALUE_OFFSET) + as *const i64) + }; + let payload_fn = match op_code { + OpCode::IntAddOvf => pyre_object::longobject::jit_bigint_add_int_int as *const (), + OpCode::IntSubOvf => pyre_object::longobject::jit_bigint_sub_int_int as *const (), + OpCode::IntMulOvf => pyre_object::longobject::jit_bigint_mul_int_int as *const (), + _ => unreachable!("overflow arm requires Add/Sub/Mul"), + }; + let concrete_args = [ + majit_ir::Value::Int(payload_fn as usize as i64), + majit_ir::Value::Int(la), + majit_ir::Value::Int(rb), + ]; + let payload = ctx.trace_ctx.call_typed_with_effect_pure_can_raise( + OpCode::CallR, + payload_fn, + &[lhs_raw, rhs_raw], + &[majit_ir::Type::Int, majit_ir::Type::Int], + majit_ir::Type::Ref, + majit_metainterp::ELIDABLE_OR_MEMERROR_EFFECT_INFO, + &concrete_args, + majit_ir::Value::Ref(majit_ir::GcRef(payload_concrete as usize)), + ); + ctx.trace_ctx.set_opref_concrete( + payload, + majit_ir::Value::Ref(majit_ir::GcRef(payload_concrete as usize)), + ); + if payload.inline_const_to_value().is_none() { + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoException, &[])?; + } + + let fits_fn = pyre_object::longobject::jit_bigint_fits_int as *const (); + let fits = ctx.trace_ctx.call_typed_with_effect( + OpCode::CallI, + fits_fn, + &[payload], + &[majit_ir::Type::Ref], + majit_ir::Type::Int, + majit_metainterp::cannot_raise_effect_info(), + ); + ctx.trace_ctx + .set_opref_concrete(fits, majit_ir::Value::Int(0)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardFalse, &[fits])?; + + let result = crate::helpers::emit_box_long_inline( + ctx.trace_ctx, + payload, + crate::descr::w_long_size_descr(), + crate::descr::long_value_descr(), + ); + ctx.trace_ctx.set_opref_concrete( + result, + majit_ir::Value::Ref(majit_ir::GcRef(boxed_result_i64 as usize)), + ); + write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; + return Ok(Some(())); + } let (raw_result, concrete_value) = match op_code { OpCode::IntFloorDiv | OpCode::IntMod => { // rint.py _ovf_zer guards: int_eq(rhs,0)→guard_false + @@ -606,12 +682,14 @@ pub(crate) fn try_walker_specialize_binary_op_long( dst: usize, dst_bank: char, ) -> Result, DispatchError> { + use pyre_interpreter::bytecode::BinaryOperator; if !ctx.is_authoritative_executor || r_args.len() != 2 || dst_bank != 'r' { return Ok(None); } - let Some(spec) = pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag) - .and_then(crate::trace_opcode::long_binop_raw_helper) - else { + let Some(op) = pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag) else { + return Ok(None); + }; + let Some(spec) = crate::trace_opcode::long_binop_raw_helper(op) else { return Ok(None); }; let lhs = r_args[0]; @@ -630,16 +708,17 @@ pub(crate) fn try_walker_specialize_binary_op_long( let Some(boxed_result_i64) = walker_execute_may_force_boxed(ctx, allboxes, call_descr) else { return Ok(None); }; - // Pyre representation demote: when the bigint result fits i64 it becomes a - // W_IntObject in pyre's two-class int model, which the inline-NEW long box - // cannot represent — so decline the spec here (before emitting any op) and - // let the generic record handle the demote. Reuse the authentic boxed - // result's payload instead of running `spec.raw_fn` a second time; the raw - // helpers allocate/publish exception state and must not be used as a - // trace-time probe. + // A NULL result means the op raised — defer to the generic record. `newlong` + // never demotes, so an arithmetic long op always yields a W_LongObject the + // inline-NEW box below can represent. The shift ops are the only ones that + // can still yield a W_IntObject (`space.newint(-1)`/`(0)` on a shift count + // that overflows a machine int); the `!is_long` decline routes that + // huge-count case to the generic leg. Reuse the authentic boxed result's + // payload instead of running `spec.raw_fn` a second time; the raw helpers + // allocate/publish exception state and must not be used as a trace-time + // probe. let boxed_result_obj = boxed_result_i64 as usize as pyre_object::PyObjectRef; - if boxed_result_obj == pyre_object::PY_NULL || unsafe { pyre_object::is_int(boxed_result_obj) } - { + if boxed_result_obj == pyre_object::PY_NULL { return Ok(None); } if !unsafe { pyre_object::is_long(boxed_result_obj) } { @@ -649,7 +728,6 @@ pub(crate) fn try_walker_specialize_binary_op_long( *((boxed_result_obj as *const u8).add(pyre_object::longobject::LONG_VALUE_OFFSET) as *const i64) }; - let fits_concrete = 0_i64; let long_type_addr = &pyre_object::pyobject::LONG_TYPE as *const _ as i64; walker_guard_class(ctx, op_pc, lhs, long_type_addr)?; walker_guard_class(ctx, op_pc, rhs, long_type_addr)?; @@ -708,22 +786,33 @@ pub(crate) fn try_walker_specialize_binary_op_long( if raw.inline_const_to_value().is_none() { walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoException, &[])?; } - // `newlong` demote guard: `GuardFalse(fits_int(raw))`. Passes at record time - // (checked above), deopts to the interpreter if a future replay yields an - // i64-fitting result. Resumes at op_pc (the BINARY_OP), like the GuardClass - // guards. - let fits_fn = pyre_object::longobject::jit_bigint_fits_int as *const (); - let fits = ctx.trace_ctx.call_typed_with_effect( - OpCode::CallI, - fits_fn, - &[raw], - &[majit_ir::Type::Ref], - majit_ir::Type::Int, - majit_metainterp::cannot_raise_effect_info(), - ); - ctx.trace_ctx - .set_opref_concrete(fits, majit_ir::Value::Int(fits_concrete)); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardFalse, &[fits])?; + // Shift-count demote guard: `_lshift`/`_rshift` demote to `space.newint` + // (-1/0) when the shift count overflows a machine int (`toint()` + // OverflowError), so guard that the count fits and let a huge-count replay + // deopt to the generic leg. The arithmetic ops (`newlong`, no demote) emit + // no such guard — a fitting result stays a W_LongObject in the trace. + if matches!( + op, + BinaryOperator::Lshift + | BinaryOperator::InplaceLshift + | BinaryOperator::Rshift + | BinaryOperator::InplaceRshift + ) { + let fits_fn = pyre_object::longobject::jit_bigint_fits_int as *const (); + let count_fits = ctx.trace_ctx.call_typed_with_effect( + OpCode::CallI, + fits_fn, + &[rhs_pl], + &[majit_ir::Type::Ref], + majit_ir::Type::Int, + majit_metainterp::cannot_raise_effect_info(), + ); + let count_fits_concrete = + unsafe { pyre_object::longobject::jit_bigint_fits_int(rhs_payload) }; + ctx.trace_ctx + .set_opref_concrete(count_fits, majit_ir::Value::Int(count_fits_concrete)); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[count_fits])?; + } // Inline `W_LongObject(raw)` NEW (`new_with_vtable` + `setfield_gc('value')`). // NewWithVtable lowers to the collecting `CallMallocNursery` — the GC // safepoint that lets bigint-heavy loops reclaim dead bigints. @@ -1619,8 +1708,10 @@ pub(crate) fn try_walker_specialize_store_attr( /// byte-correct) for any shape it cannot reproduce faithfully: empty list /// (Empty strategy), a non-const / unrecoverable array length, an element /// without a concrete Ref shadow, or an Integer-strategy list that carries a -/// fits-in-word `W_LongObject` / tagged immediate (which `walker_unbox_int`'s -/// `&INT_TYPE` guard does not cover). +/// tagged immediate (which has no `&INT_TYPE`/`&LONG_TYPE` header for the +/// unbox guard). A fits-in-word `W_LongObject` is accepted: `is_plain_int1` +/// covers it and `walker_unbox_long` supplies the `&LONG_TYPE` + `_fits_int` +/// guarded extraction. pub(crate) fn try_walker_specialize_newlist( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, @@ -1685,18 +1776,21 @@ pub(crate) fn try_walker_specialize_newlist( // (a minor collection there could move the boxed elements, so the raw // pointers must not be dereferenced afterwards). enum Emit { - Int(Vec), + // Per element: `(unboxed i64, is_fits_long)`. `is_fits_long` selects + // `walker_unbox_long` (`&LONG_TYPE` + `_fits_int` guard) over the + // plain `walker_unbox_int`. + Int(Vec<(i64, bool)>), Float(Vec), Object, } let int_ty = &pyre_object::pyobject::INT_TYPE as *const pyre_object::pyobject::PyType; let emit = match strategy { ListStrategy::Integer => { - // `list_strategy_for` accepts fits-in-word `W_LongObject` and - // tagged immediates under Integer, but `walker_unbox_int` only - // covers the exact `W_IntObject` (`&INT_TYPE` + `intval`) shape — - // decline otherwise so the residual (correct for any element) - // rebuilds the same Integer list. + // `IntegerListStrategy.is_correct_type` is `is_plain_int1`, which + // accepts an exact `W_IntObject` or a fits-in-word `W_LongObject`; + // both store the unboxed i64 (`plain_int_w`). A tagged immediate + // has no header for the unbox guard, so decline it to the residual + // (correct for any element). let mut vals = Vec::with_capacity(len); for &p in &concretes { if pyre_object::tagged_int::CAN_BE_TAGGED @@ -1704,12 +1798,16 @@ pub(crate) fn try_walker_specialize_newlist( { return Ok(None); } - let exact_int = - unsafe { pyre_object::is_plain_int1(p) && std::ptr::eq((*p).ob_type, int_ty) }; - if !exact_int { + if !unsafe { pyre_object::is_plain_int1(p) } { return Ok(None); } - vals.push(unsafe { pyre_object::w_int_get_value(p) }); + let is_fits_long = unsafe { pyre_object::pyobject::is_long(p) }; + let val = if is_fits_long { + pyre_object::longobject::jit_w_long_toint(p as usize as i64) + } else { + unsafe { pyre_object::w_int_get_value(p) } + }; + vals.push((val, is_fits_long)); } Emit::Int(vals) } @@ -1740,9 +1838,14 @@ pub(crate) fn try_walker_specialize_newlist( let list_op = match emit { Emit::Int(vals) => { let int_type_addr = int_ty as i64; + let long_type_addr = &pyre_object::pyobject::LONG_TYPE as *const _ as i64; let mut raws: Vec = Vec::with_capacity(len); - for (&it, &v) in items.iter().zip(vals.iter()) { - let raw = walker_unbox_int(ctx, op_pc, it, int_type_addr)?; + for (&it, &(v, is_fits_long)) in items.iter().zip(vals.iter()) { + let raw = if is_fits_long { + walker_unbox_long(ctx, op_pc, it, long_type_addr)? + } else { + walker_unbox_int(ctx, op_pc, it, int_type_addr)? + }; ctx.trace_ctx .set_opref_concrete(raw, majit_ir::Value::Int(v)); raws.push(raw); @@ -1799,8 +1902,8 @@ pub(crate) fn try_walker_specialize_newlist( /// /// Returns `Ok(Some(()))` when folded (the caller returns `Continue`); /// `Ok(None)` to fall through to the opaque residual, which stays correct -/// for any other shape (object tuple, arity ≠ 2, long element, cache miss) -/// — so a non-foldable build is not declined. +/// for any other shape (object tuple, arity ≠ 2, out-of-range long, tagged +/// immediate, cache miss) — so a non-foldable build is not declined. pub(crate) fn try_walker_specialize_newtuple( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, @@ -1844,61 +1947,72 @@ pub(crate) fn try_walker_specialize_newtuple( ) else { return Ok(None); }; - // Only the arity-2 plain-int specialised tuple is folded today. Gate - // on `is_plain_int1` (rejects int subclasses + non-fitting longs) AND - // an exact `&INT_TYPE` `ob_type` — that excludes the fits-in-word - // `W_LongObject` arm `is_plain_int1` also accepts, which would need the - // long unbox the retired trait-side payload helper did (out of scope - // here). Any other shape falls through to the residual (correct). + // The arity-2 int specialised tuple `Cls_ii` (`makespecialisedtuple2`, + // specialisedtupleobject.py) is built when both elements pass + // `is_plain_int1` — an exact `W_IntObject` or a fits-in-word + // `W_LongObject`; the stored payload is `plain_int_w` of each. A tagged + // immediate has no real header for the unbox guard and the emit is not + // tag-aware, so decline it to the residual (correct for any shape). if pyre_object::tagged_int::CAN_BE_TAGGED && (pyre_object::tagged_int::is_tagged_int(c0) || pyre_object::tagged_int::is_tagged_int(c1)) { - // A tagged-immediate element has no real header to read for the exact - // `&INT_TYPE` ob_type check below, and the spec_ii emit (w_class guard + - // typed unbox) is not tag-aware. Fall through to the opaque residual, - // which is correct for any element shape. return Ok(None); } let int_ty = &pyre_object::pyobject::INT_TYPE as *const pyre_object::pyobject::PyType; - let both_plain_int = unsafe { - pyre_object::is_plain_int1(c0) - && pyre_object::is_plain_int1(c1) - && std::ptr::eq((*c0).ob_type, int_ty) - && std::ptr::eq((*c1).ob_type, int_ty) - }; + let both_plain_int = + unsafe { pyre_object::is_plain_int1(c0) && pyre_object::is_plain_int1(c1) }; if !both_plain_int { return Ok(None); } - // Concrete element int payloads (already proven plain `W_IntObject`). - let (v0, v1) = unsafe { - ( - pyre_object::w_int_get_value(c0), - pyre_object::w_int_get_value(c1), - ) + let c0_long = unsafe { pyre_object::pyobject::is_long(c0) }; + let c1_long = unsafe { pyre_object::pyobject::is_long(c1) }; + // Concrete element int payloads (`plain_int_w`: `W_IntObject`'s `intval` + // or a fits-int `W_LongObject`'s `toint()`). + let v0 = if c0_long { + pyre_object::longobject::jit_w_long_toint(c0 as usize as i64) + } else { + unsafe { pyre_object::w_int_get_value(c0) } + }; + let v1 = if c1_long { + pyre_object::longobject::jit_w_long_toint(c1 as usize as i64) + } else { + unsafe { pyre_object::w_int_get_value(c1) } }; // --- emit the virtual spec_ii walker-native --- // Paired `w_class` guard per element so a runtime int subclass sharing - // `&INT_TYPE`'s payload side-exits, then the plain-int payload unbox. + // the public `int` `w_class` side-exits, then the plain-int payload unbox. + // A fits-int `W_LongObject` also carries the public `int` `w_class` + // (`is_plain_int1`), so the same guard covers it; the payload extraction + // switches to `walker_unbox_long` (`&LONG_TYPE` + `_fits_int`). let int_typeobj = pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::INT_TYPE); walker_guard_exact_w_class(ctx, op_pc, e0, int_typeobj)?; walker_guard_exact_w_class(ctx, op_pc, e1, int_typeobj)?; let int_type_addr = int_ty as i64; - let raw0 = walker_unbox_int_typed( - ctx, - op_pc, - e0, - int_type_addr, - crate::descr::int_intval_descr(), - )?; - let raw1 = walker_unbox_int_typed( - ctx, - op_pc, - e1, - int_type_addr, - crate::descr::int_intval_descr(), - )?; + let long_type_addr = &pyre_object::pyobject::LONG_TYPE as *const _ as i64; + let raw0 = if c0_long { + walker_unbox_long(ctx, op_pc, e0, long_type_addr)? + } else { + walker_unbox_int_typed( + ctx, + op_pc, + e0, + int_type_addr, + crate::descr::int_intval_descr(), + )? + }; + let raw1 = if c1_long { + walker_unbox_long(ctx, op_pc, e1, long_type_addr)? + } else { + walker_unbox_int_typed( + ctx, + op_pc, + e1, + int_type_addr, + crate::descr::int_intval_descr(), + )? + }; let tuple = ctx.trace_ctx.record_op_with_descr( OpCode::NewWithVtable, @@ -3253,16 +3367,15 @@ unsafe fn orthodox_list_append_recognize( inner_self: pyre_object::PyObjectRef, value: pyre_object::PyObjectRef, ) -> Option { - // `is_plain_int1` accepts a fits-int `W_LongObject` (it implies - // `_fits_int()`), but a long is declined here: the commit path pins - // `guard_class(value, INT_TYPE)` and a long has `ob_type == LONG_TYPE`, - // so supporting it needs equivalent `unbox_long` machinery - // (guard_class LONG_TYPE + `_fits_int` residual guard + long - // extraction) threaded through the sub-walk (PR248 §2). Empirically a - // fits-int `W_LongObject` does not reach this append: pyre normalizes - // fits-int results to `W_IntObject` across arithmetic / `int(str)` / - // literals, so the long arm is an unreachable optimization and the - // decline is correctness-safe (the generic residual handles it). + // `is_plain_int1` accepts an exact `W_IntObject` or a fits-int + // `W_LongObject`; both route to Integer storage. The commit path pins + // `guard_class(value, LONG_TYPE)` for a long value (vs `INT_TYPE` for an + // int) so the descended `w_list_append` body observes the right `ob_type`. + // The body's `is_plain_int1(value)` / `plain_int_w(value)` then unbox the + // long through the compiled `_fits_int` / `toint` path; when that path + // reaches a helper the sub-walk cannot lower it declines + // (`OrthodoxSubWalkTraceUnsupported`) and rolls back to the generic + // residual (correctness-safe for any element). if !pyre_object::pyobject::is_list(inner_self) { return None; } @@ -3276,15 +3389,14 @@ unsafe fn orthodox_list_append_recognize( return None; } let int_ok = pyre_object::is_plain_int1(value) - && !pyre_object::pyobject::is_long(value) && !(pyre_object::tagged_int::CAN_BE_TAGGED && pyre_object::tagged_int::is_tagged_int(value)); let float_ok = !value.is_null() && pyre_object::is_plain_float_strict(value); - // switch_to_correct_strategy routes `is_plain_int1` -> Integer with no - // tagged exclusion. Exclude any plain-int / float from the object - // fallback so a tagged-int / fits-int `W_LongObject` DECLINES (generic - // residual) instead of mis-routing to Object and diverging the traced - // strategy from the concrete one the commit installs. + // switch_to_correct_strategy routes `is_plain_int1` (exact int or + // fits-in-word long) -> Integer with no tagged exclusion. Exclude any + // plain-int / float from the object fallback so a tagged-int DECLINES + // (generic residual) instead of mis-routing to Object and diverging the + // traced strategy from the concrete one the commit installs. let obj_ok = !value.is_null() && !pyre_object::is_plain_int1(value) && !pyre_object::is_plain_float_strict(value); @@ -3297,13 +3409,12 @@ unsafe fn orthodox_list_append_recognize( if !pyre_object::w_list_can_append_without_realloc(inner_self) { return None; } - // Int-storage specialization: plain-int value stored unboxed (a - // fits-int `W_LongObject` is declined, see note above). - // A tagged-immediate value would need a tag-aware unboxed store and no - // `w_class` pin; decline to the generic residual append instead. + // Int-storage specialization: `is_plain_int1` value (exact `W_IntObject` + // or fits-int `W_LongObject`) stored unboxed. A tagged-immediate value + // would need a tag-aware unboxed store and no `w_class` pin; decline to + // the generic residual append instead. let int_ok = pyre_object::w_list_uses_int_storage(inner_self) && pyre_object::is_plain_int1(value) - && !pyre_object::pyobject::is_long(value) && !(pyre_object::tagged_int::CAN_BE_TAGGED && pyre_object::tagged_int::is_tagged_int(value)); // Object-storage extension: any non-null `Ref` value stored into the @@ -3410,7 +3521,6 @@ pub(crate) fn orthodox_list_append_commit( if promote_empty { let target = unsafe { let int_ok = pyre_object::is_plain_int1(value) - && !pyre_object::pyobject::is_long(value) && !(pyre_object::tagged_int::CAN_BE_TAGGED && pyre_object::tagged_int::is_tagged_int(value)); if int_ok { @@ -3447,25 +3557,31 @@ pub(crate) fn orthodox_list_append_commit( } // Pin the appended value's class so the inlined `is_plain_int1` type - // predicate folds during the sub-walk: guard_class(value, INT_TYPE) + - // class_now_known, so its `is_int`/`is_bool` typeptr reads fold to the - // INT_TYPE const (the typeptr fold in `getfield_gc_via_heapcache`). The - // recognition gate already proved `is_plain_int1(value)`; this guard - // enforces ob_type==INT_TYPE at runtime. The value's integer payload + // predicate folds during the sub-walk: guard_class(value, ) + + // class_now_known, so its `is_int`/`is_long`/`is_bool` typeptr reads fold + // to the pinned const (the typeptr fold in `getfield_gc_via_heapcache`). + // The recognition gate already proved `is_plain_int1(value)`; this guard + // enforces the observed ob_type at runtime. The value's integer payload // stays symbolic — only its class is pinned. // // Object-storage append stores the value as a // plain GC ref with no unboxing, so it carries no type precondition — - // skip the INT_TYPE pin (the sub-walk's object-storage store path does + // skip the class pin (the sub-walk's object-storage store path does // not read the value's class). let is_obj_storage = unsafe { pyre_object::w_list_uses_object_storage(inner_self) }; if !is_obj_storage { // Integer and Float storage both pin the value's class so the body's - // strict type test folds during the sub-walk; only the ob_type const - // differs (INT_TYPE vs FLOAT_TYPE). + // strict type test folds during the sub-walk; the ob_type const is + // FLOAT_TYPE for float storage, and INT_TYPE / LONG_TYPE for int + // storage depending on whether the value is an exact int or a fits-int + // `W_LongObject` (both pass `is_plain_int1` -> Integer storage, but + // carry distinct `ob_type`s the sub-walk's `is_plain_int1` folds on). let is_float_storage = unsafe { pyre_object::w_list_uses_float_storage(inner_self) }; + let value_is_long = unsafe { pyre_object::pyobject::is_long(value) }; let value_type_addr = if is_float_storage { &pyre_object::pyobject::FLOAT_TYPE as *const _ as i64 + } else if value_is_long { + &pyre_object::pyobject::LONG_TYPE as *const _ as i64 } else { &pyre_object::pyobject::INT_TYPE as *const _ as i64 }; @@ -5643,17 +5759,20 @@ pub(crate) fn try_walker_store_name_cell_fold( if majit_gc::can_move(majit_ir::GcRef(stored as usize)) { return Ok(false); } - // The stored value must be a provably-plain-int box. `is_plain_int1` on - // the trace-time concrete rejects `bool` / int-subclass / `long` (whose - // `write_cell` replaces the cell rather than mutating `intvalue`); the - // heapcache lookup recovers the box's raw `intvalue` (populated only by - // JIT int boxes, `emit_box_int_inline`), so the setfield needs no runtime - // class guard — exactly as pypy's optimized trace folds the - // `is_plain_int1` check away for an `int_add` result. + // The stored value must be a provably-plain-int box. `is_plain_int1` accepts + // a fits-int `W_LongObject`, whose `write_cell` REPLACES the cell rather than + // mutating `intvalue`, so exclude `long` explicitly; the remaining int box's + // raw `intvalue` (populated only by JIT int boxes, `emit_box_int_inline`) is + // recovered by the heapcache lookup, so the setfield needs no runtime class + // guard — exactly as pypy's optimized trace folds the `is_plain_int1` check + // away for an `int_add` result. (bool / int-subclass are already excluded by + // `is_plain_int1`.) let is_plain_int = matches!( ctx.trace_ctx.box_value(value_opref), Some(majit_ir::Value::Ref(majit_ir::GcRef(p))) - if p != 0 && unsafe { pyre_object::listobject::is_plain_int1(p as pyre_object::PyObjectRef) } + if p != 0 + && unsafe { pyre_object::listobject::is_plain_int1(p as pyre_object::PyObjectRef) } + && !unsafe { pyre_object::is_long(p as pyre_object::PyObjectRef) } ); if !is_plain_int { return Ok(false); diff --git a/pyre/pyre-jit-trace/src/trace_opcode.rs b/pyre/pyre-jit-trace/src/trace_opcode.rs index e69f04b771f..d333aa415f5 100644 --- a/pyre/pyre-jit-trace/src/trace_opcode.rs +++ b/pyre/pyre-jit-trace/src/trace_opcode.rs @@ -261,8 +261,10 @@ fn trace_abort_error(reason: &'static str) -> PyError { /// The elidable `rbigint` payload helper + effect for a walker-specialised /// W_LongObject binary op (see [`long_binop_raw_helper`]). The bigint result is -/// boxed by the caller as a `W_LongObject` after the pyre-specific fits-int -/// demotion guard. +/// boxed by the caller as a `W_LongObject` unconditionally (`newlong` never +/// demotes); the shift ops additionally guard that the shift count fits a +/// machine int, deopting a huge count to the generic leg (which produces the +/// `space.newint` W_IntObject). /// True-divide is NOT here — it returns a float (`CallPureF` + `wrapfloat`), so /// it has its own specialisation ([`try_walker_specialize_truediv_op_long`]). pub(crate) struct LongBinopSpec { diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index 83e28c86acf..b67aa806aa7 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -13033,14 +13033,17 @@ impl CodeWriter { spliced.insns = new_insns; } // Per-PC leading `-live-` (marker PRESENCE), scoped to FOR_ITER body - // PCs. `jtransform.py` puts a `-live-` before every deopt-capable op; - // pyre emits `-live-` only trailing-after-calls, so a FOR_ITER body - // guard has no marker of its own: `derive_pc_live_indices_from_sparse` - // rounds its PC back to the header marker, whose resume coordinate is - // not the body's. Give each FOR_ITER-body pc-carrying op its own - // leading marker so its PC resolves to itself. While-loop body PCs - // are excluded — their existing header-folded marker is correct and - // giving them individual markers changes their resume layout, breaking + // PCs and attribute opcodes. `jtransform.py` puts a `-live-` at each + // guard resume point (`rpython/jit/codewriter/flatten.py:258-260, + // 282-286`). Pyre otherwise emits `-live-` only trailing-after-calls, + // so a FOR_ITER body guard or a LoadAttr/StoreAttr specialization + // guard can have no marker of its own: + // `derive_pc_live_indices_from_sparse` rounds its PC back to the + // preceding marker, whose Python coordinate and stack depth belong to + // another opcode. Give these pc-carrying ops their own leading + // marker so their snapshots resume at the opcode they re-execute. + // Other while-loop body PCs remain excluded — giving all of them + // individual markers changes their resume layout and regresses // nbody/nested_loop/spectral_norm. { // Build the set of FOR_ITER body PCs: py_pc in @@ -13072,7 +13075,15 @@ impl CodeWriter { if py_pc < 0 { continue; } - if !foriter_body_pcs.contains(py_pc as usize) { + let py_pc = py_pc as usize; + let is_attr = matches!( + pyre_interpreter::decode_instruction_at(code, py_pc), + Some(( + Instruction::LoadAttr { .. } | Instruction::StoreAttr { .. }, + _ + )) + ); + if !foriter_body_pcs.contains(py_pc) && !is_attr { continue; } let already = pos diff --git a/pyre/pyre-object/src/longobject.rs b/pyre/pyre-object/src/longobject.rs index 35474ceb5fc..1bb2ac36d5c 100644 --- a/pyre/pyre-object/src/longobject.rs +++ b/pyre/pyre-object/src/longobject.rs @@ -154,21 +154,34 @@ pub unsafe fn bigint_external_size(addr: usize) -> usize { pub fn alloc_bigint_nursery_collecting(value: BigInt) -> *mut BigInt { let tid = bigint_gc_type_id(); if tid != 0 { + // A few limbs of external bytes are noise next to the 48-byte struct + // the nursery already tracks; skip both charge crossings for them so + // small-bignum churn stays on the plain bump-alloc path. The + // end-of-major recompute absorbs the drift for survivors. let external = bigint_external_bytes(&value); - crate::gc_hook::try_gc_charge_memory_pressure(external); + let charge = external > SMALL_EXTERNAL_EXEMPT_BYTES; + if charge { + crate::gc_hook::try_gc_charge_memory_pressure(external); + } if let Some(raw) = crate::gc_hook::try_gc_alloc_collecting(tid, BIGINT_PAYLOAD_SIZE) .filter(|p| !p.is_null()) { unsafe { std::ptr::write(raw as *mut BigInt, value); } - crate::gc_hook::try_gc_charge_oldgen_external(raw as usize, external); + if charge { + crate::gc_hook::try_gc_charge_oldgen_external(raw as usize, external); + } return raw as *mut BigInt; } } alloc_bigint_nursery(value) } +/// External-byte threshold below which [`alloc_bigint_nursery_collecting`] +/// skips the memory-pressure / old-gen charge crossings (8 limbs). +const SMALL_EXTERNAL_EXEMPT_BYTES: usize = 64; + /// Allocate `value` as a GC-managed `BigInt` at a stable (old-gen, non-moving) /// address, for host/interpreter callers (`w_long_new`) that hold the pointer /// on the Rust stack without rooting it. Mirrors `w_float_new`'s @@ -519,14 +532,50 @@ pub extern "C" fn jit_w_long_xor_raw(a: i64, b: i64) -> i64 { #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_add(a: i64, b: i64) -> i64 { let (a, b) = (a as *const BigInt, b as *const BigInt); - unsafe { alloc_bigint_nursery_collecting(&*a + &*b) as i64 } + // Two-limb fast path: when both operands and the sum fit i128 the result + // is exact without the general limb machinery. + unsafe { + if let (Ok(x), Ok(y)) = (i128::try_from(&*a), i128::try_from(&*b)) { + if let Some(z) = x.checked_add(y) { + return alloc_bigint_nursery_collecting(BigInt::from(z)) as i64; + } + } + alloc_bigint_nursery_collecting(&*a + &*b) as i64 + } +} + +/// `rbigint.add_int_int_bigint_result` (`rpython/rlib/rbigint.py:717`, +/// `@jit.elidable`) — exact bigint sum of two machine ints. Allocates the +/// result via the COLLECTING nursery, matching [`jit_bigint_add`], and returns +/// a freshly heap-allocated `*mut BigInt` payload (as i64). +#[majit_macros::elidable_or_memerror] +pub extern "C" fn jit_bigint_add_int_int(a: i64, b: i64) -> i64 { + // Exact in i128 for any i64 pair; skips the general bigint add machinery. + unsafe { alloc_bigint_nursery_collecting(BigInt::from(a as i128 + b as i128)) as i64 } } /// `rbigint.sub` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_sub(a: i64, b: i64) -> i64 { let (a, b) = (a as *const BigInt, b as *const BigInt); - unsafe { alloc_bigint_nursery_collecting(&*a - &*b) as i64 } + // Two-limb fast path mirroring `jit_bigint_add`. + unsafe { + if let (Ok(x), Ok(y)) = (i128::try_from(&*a), i128::try_from(&*b)) { + if let Some(z) = x.checked_sub(y) { + return alloc_bigint_nursery_collecting(BigInt::from(z)) as i64; + } + } + alloc_bigint_nursery_collecting(&*a - &*b) as i64 + } +} + +/// `rbigint.sub_int_int_bigint_result` (`rpython/rlib/rbigint.py:788`, +/// `@jit.elidable`) — exact bigint difference of two machine ints. See +/// [`jit_bigint_add_int_int`]. +#[majit_macros::elidable_or_memerror] +pub extern "C" fn jit_bigint_sub_int_int(a: i64, b: i64) -> i64 { + // Exact in i128 for any i64 pair; skips the general bigint sub machinery. + unsafe { alloc_bigint_nursery_collecting(BigInt::from(a as i128 - b as i128)) as i64 } } /// `rbigint.mul` on bare payloads (collecting). See [`jit_bigint_add`]. @@ -536,6 +585,15 @@ pub extern "C" fn jit_bigint_mul(a: i64, b: i64) -> i64 { unsafe { alloc_bigint_nursery_collecting(&*a * &*b) as i64 } } +/// `rbigint.mul_int_int_bigint_result` (`rpython/rlib/rbigint.py:873`, +/// `@jit.elidable`) — exact bigint product of two machine ints. See +/// [`jit_bigint_add_int_int`]. +#[majit_macros::elidable_or_memerror] +pub extern "C" fn jit_bigint_mul_int_int(a: i64, b: i64) -> i64 { + // A 64x64 product is exact in i128; skips the general bigint mul machinery. + unsafe { alloc_bigint_nursery_collecting(BigInt::from(a as i128 * b as i128)) as i64 } +} + /// `rbigint.and_` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_and(a: i64, b: i64) -> i64 { @@ -580,32 +638,6 @@ pub extern "C" fn jit_w_long_cmp(a: i64, b: i64) -> i64 { } } -/// `bigint_result` — wrap the bigint produced by [`jit_w_long_add_raw`] in a -/// Python int, demoting to `W_IntObject` when it fits in i64, otherwise -/// reusing the `*mut BigInt` payload in a fresh `W_LongObject`. This is the -/// `W_LongObject(...)` wrapper allocation that upstream keeps a residual `NEW` -/// outside the elidable `rbigint.add` (the int fast path boxes the same way, -/// via the `dont_look_inside` `jit_w_int_new`). Marked `dont_look_inside`, not -/// elidable, so the wrapper object is never pure-CSE'd and each add yields a -/// distinct boxed result, matching `W_LongObject(op(...))`. -/// -/// The i64-range demotion to `W_IntObject` is pyre's two-class `int` -/// representation (small-int fast object + bigint object); PyPy's default -/// `newlong` (`longobject.py:495`, `withsmalllong=False`) keeps a -/// `W_LongObject`. Both denote the same `int` value — this is a representation -/// choice spanning every int path, not specific to this helper. -#[majit_macros::dont_look_inside] -pub extern "C" fn jit_bigint_result_box(num: i64) -> i64 { - let num = num as *mut BigInt; - unsafe { - if jit_bigint_to_i64_fits(&*num) != 0 { - crate::intobject::w_int_new(jit_bigint_to_i64_value(&*num)) as usize as i64 - } else { - w_long_from_raw(num) as usize as i64 - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -715,30 +747,15 @@ mod tests { } #[test] - fn test_jit_bigint_result_box_keeps_long_out_of_range() { - // Sum out of i64 range boxes as W_LongObject, reusing the payload. - let a = w_long_new(BigInt::from(i64::MAX)); - let b = w_long_new(BigInt::from(i64::MAX)); - let raw = jit_w_long_add_raw(a as i64, b as i64); - let r = jit_bigint_result_box(raw) as PyObjectRef; - unsafe { - assert!(is_long(r)); - assert_eq!(*w_long_get_value(r), BigInt::from(i64::MAX) * 2); - } - } - - #[test] - fn test_jit_bigint_result_box_demotes_to_int_when_fits() { - // `bigint_result` parity: a sum that fits in i64 demotes to W_IntObject - // (so a later GuardClass(LONG_TYPE) on the result correctly side-exits). + fn test_jit_w_long_add_raw_keeps_payload_when_fits() { + // The raw helper never demotes: a sum that fits i64 still yields a + // `*mut BigInt` payload (the boxing NEW wraps it as a W_LongObject, + // matching `newlong` which does not demote). let a = w_long_new(BigInt::from(i64::MAX) + BigInt::from(1)); let b = w_long_new(BigInt::from(-1) - BigInt::from(i64::MAX)); - let raw = jit_w_long_add_raw(a as i64, b as i64); - let r = jit_bigint_result_box(raw) as PyObjectRef; + let raw = jit_w_long_add_raw(a as i64, b as i64) as *mut BigInt; unsafe { - assert!(is_int(r)); - assert!(!is_long(r)); - assert_eq!(crate::intobject::w_int_get_value(r), 0); + assert_eq!(*raw, BigInt::from(0)); } } }