diff --git a/AGENTS.md b/AGENTS.md index d9b7dfae2eb..cb790ffdc17 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -233,6 +233,13 @@ comment at the site citing both sides. - Confirm the worktree (`git rev-parse --show-toplevel`) before editing and before staging — dozens of sibling worktrees share one `.git`. +## Build and CI polling + +- Never check a running local build or CI job more often than once every five + minutes. While it runs, continue useful independent work when any is + available; otherwise wait for the five-minute boundary. Do not spend tokens + on repetitive status-only updates. + ## Before committing - `cargo test --all --no-default-features --features dynasm`. Both halves matter. diff --git a/majit/majit-backend-dynasm/src/lib.rs b/majit/majit-backend-dynasm/src/lib.rs index cf2e913badc..7eb9049ea55 100644 --- a/majit/majit-backend-dynasm/src/lib.rs +++ b/majit/majit-backend-dynasm/src/lib.rs @@ -635,6 +635,38 @@ fn handle_fail_propagate_exception(frame_ptr: *mut jitframe::JitFrame) -> i64 { value } +/// Read a guard's logical fail arguments from its physical JITFRAME slots. +/// +/// `llsupport/assembler.py::store_info_on_descr` writes `0xFFFF` for a +/// resume-data hole, and `ResumeDataDirectReader.decode_ref` (plus the other +/// typed decoders) asks the CPU for a physical slot only in its TAGBOX arm. +/// TAGCONST/TAGVIRTUAL values are reconstructed from resume metadata instead. +/// Majit's [`guard::decode_rd_loc_slot`] represents the sentinel as `None`; +/// use zero as the inert carrier, exactly as +/// `DynasmBackend::execute_token_ints_raw` does. Only a synthetic +/// descriptor with no `rd_locs` entry at all retains the historical +/// identity-slot fallback used by backend tests. +unsafe fn guard_fail_values( + descr: &dyn majit_ir::FailDescr, + frame_ptr: *mut jitframe::JitFrame, +) -> Vec { + let rd_locs_len = descr.rd_locs().len(); + descr + .fail_arg_types() + .iter() + .enumerate() + .map(|(i, _)| { + if i < rd_locs_len { + guard::decode_rd_loc_slot(descr, i) + .map(|slot| unsafe { llmodel::get_int_value_direct(frame_ptr, slot) as i64 }) + .unwrap_or(0) + } else { + unsafe { llmodel::get_int_value_direct(frame_ptr, i) as i64 } + } + }) + .collect() +} + /// compile.py `AbstractResumeGuardDescr.handle_fail`. /// /// Upstream: @@ -682,16 +714,12 @@ fn handle_fail_resume_guard( ) -> i64 { let trace_id = descr.trace_id(); let fail_index = descr.fail_index_per_trace(); - let n_fail_args = descr.fail_arg_types().len(); - let mut raw_values: Vec = Vec::with_capacity(n_fail_args); - for i in 0..n_fail_args { - // PyPy `llmodel.py _decode_pos` parity: read the slot - // from `descr.rd_locs[i]`. Synthetic descrs without `rd_locs` - // fall back to identity slot indexing — same shape as the - // pre-Slice-MM table-miss path. - let slot = guard::decode_rd_loc_slot(descr, i).unwrap_or(i); - raw_values.push(unsafe { llmodel::get_int_value_direct(frame_ptr, slot) as i64 }); - } + // PyPy `llsupport/assembler.py::store_info_on_descr` plus + // `ResumeDataDirectReader.decode_ref` parity: a 0xFFFF entry is a + // resume-data hole, not permission to read the same-numbered physical + // slot. The old fallback turned an unrelated or uninitialised JITFRAME + // word into a Ref and registered it as a GC root during bridge recovery. + let mut raw_values = unsafe { guard_fail_values(descr, frame_ptr) }; let guard_value_operand = majit_backend::guard_value_counter_slot(descr) .map(|slot| unsafe { llmodel::get_int_value_direct(frame_ptr, slot) as i64 }); @@ -918,6 +946,46 @@ mod tests { ptr } + #[test] + fn test_guard_fail_values_do_not_read_resume_holes() { + let descr = majit_backend::make_resume_guard_descr_typed(vec![Type::Ref, Type::Int]); + let fail_descr = descr.as_fail_descr().expect("resume guard descr"); + fail_descr.set_rd_locs(vec![0xFFFF, 2]); + + // Slot 0 deliberately contains a pointer-shaped poison. The first + // logical failarg is a resume-data hole, so recovery must not publish + // or root this physical word as a Ref. The mapped second argument is + // still read from its encoded physical slot. + let jf = unsafe { alloc_test_jitframe(0, &[0x1234_5678, 11, 77]) }; + let values = unsafe { guard_fail_values(fail_descr, jf) }; + assert_eq!(values, vec![0, 77]); + + unsafe { libc::free(jf as *mut std::ffi::c_void) }; + } + + #[test] + fn test_guard_fail_values_all_holes_need_no_frame_slots() { + let descr = majit_backend::make_resume_guard_descr_typed(vec![Type::Ref; 3]); + let fail_descr = descr.as_fail_descr().expect("resume guard descr"); + fail_descr.set_rd_locs(vec![0xFFFF; 3]); + + // No physical slot exists for these logical positions. In particular, + // recovery must not perform even a speculative read of a hole. + let jf = unsafe { alloc_test_jitframe(0, &[]) }; + assert_eq!(unsafe { guard_fail_values(fail_descr, jf) }, vec![0; 3]); + unsafe { libc::free(jf as *mut std::ffi::c_void) }; + } + + #[test] + fn test_guard_fail_values_synthetic_descr_keeps_identity_slots() { + let descr = majit_backend::make_resume_guard_descr_typed(vec![Type::Int; 2]); + let fail_descr = descr.as_fail_descr().expect("resume guard descr"); + assert!(fail_descr.rd_locs().is_empty()); + let jf = unsafe { alloc_test_jitframe(0, &[11, 77]) }; + assert_eq!(unsafe { guard_fail_values(fail_descr, jf) }, vec![11, 77]); + unsafe { libc::free(jf as *mut std::ffi::c_void) }; + } + // ── Bug 1 regression: unresolved target must not dereference result as pointer ── // The old code let the helper return value flow into `mov rdx, rax; mov rcx, [rdx]` // which dereferenced an integer as a pointer. This test verifies the trampoline diff --git a/majit/majit-metainterp/src/blackhole.rs b/majit/majit-metainterp/src/blackhole.rs index 13b85cc4c72..f285303329c 100644 --- a/majit/majit-metainterp/src/blackhole.rs +++ b/majit/majit-metainterp/src/blackhole.rs @@ -10470,6 +10470,11 @@ pub fn build_inline_call_only_bh_builder() -> BlackholeInterpBuilder { "new_with_vtable/d>r", majit_translate::insns::BC_NEW_WITH_VTABLE, ), + // The header read `jtransform.rs rewrite_op_getfield` replaces, + // in the ref bank (`OpKind::GuardClass`: one ref register in, the + // class out) — emitted once a descended helper reads `ob_type` + // through a graph the codewriter looks inside. + ("guard_class/r>r", majit_translate::insns::BC_GUARD_CLASS_R), ] { insns.insert(key.to_string(), byte); } @@ -10930,6 +10935,9 @@ pub fn wire_bhimpl_handlers(builder: &mut BlackholeInterpBuilder) { // Canonical key is `guard_class/r>i`; the previous `/ri` shape was // a pyre-invented bigram that omitted the `>i` return marker. builder.wire_handler("guard_class/r>i", handler_guard_class); + // The same op with its result in the ref bank — the bank of the header + // read `jtransform.rs rewrite_op_getfield` replaced (`BC_GUARD_CLASS_R`). + builder.wire_handler("guard_class/r>r", handler_guard_class_r); // RPython `rpython/jit/metainterp/blackhole.py:1537-1539`: // @arguments("r", "d", "d") // def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr): @@ -11261,6 +11269,18 @@ fn handler_guard_class( bh.registers_i[code[p + 1] as usize] = typeptr; Ok(p + 2) } +/// [`handler_guard_class`] with the class delivered to the ref bank — +/// `guard_class/r>r`, for a header read the graph typed as a GC ref. +fn handler_guard_class_r( + bh: &mut BlackholeInterpreter, + code: &[u8], + p: usize, +) -> Result { + let cpu = bh.cpu(); + let typeptr = cpu.bh_classof(bh.registers_r[code[p] as usize]); + bh.registers_r[code[p + 1] as usize] = typeptr; + Ok(p + 2) +} /// Safe fallback for the obsolete pyre-only named vtable lookup. /// /// PyPy's `ClassRepr.getclsfield` emits an ordinary field read; it never tries diff --git a/majit/majit-metainterp/src/compile.rs b/majit/majit-metainterp/src/compile.rs index ff9b6b21153..3890a6adce3 100644 --- a/majit/majit-metainterp/src/compile.rs +++ b/majit/majit-metainterp/src/compile.rs @@ -4607,8 +4607,26 @@ impl FailDescr for ResumeGuardCopiedDescr { fn set_adr_jump_offset(&self, offset: usize) { unsafe { *self.adr_jump_offset.get() = offset }; } + /// `store_info_on_descr` writes `guardtok.faildescr.rd_locs` + /// (`llsupport/assembler.py:279`), so a backend that emits machine code for + /// this guard leaves its own physical positions here. The frontend's + /// identity-with-holes layout is resume data instead, and + /// `_copy_resume_data_from` never runs `store_final_boxes_in_guard` on a + /// copied descr, so it is written only on the donor. Read through to the + /// donor when no backend has written one, the same way `fail_arg_types`, + /// `rd_numb` and `rd_consts` chase `get_resumestorage(): return prev` + /// (`compile.py:847-850`). Without it the hole mask is empty on every + /// backend that keeps failargs in their logical slots, and both consumers + /// silently take their unmasked branch. fn rd_locs(&self) -> &[u16] { - unsafe { &*self.rd_locs.get() } + let own = unsafe { &*self.rd_locs.get() }; + if !own.is_empty() { + return own; + } + self.prev() + .as_fail_descr() + .map(|fd| fd.rd_locs()) + .unwrap_or(&[]) } fn set_rd_locs(&self, locs: Vec) { unsafe { *self.rd_locs.get() = locs }; @@ -5157,6 +5175,24 @@ pub fn copy_all_attributes_from(my_descr: &DescrRef, donor_descr: &DescrRef) { my_fd.set_rd_consts_arc(donor_fd.rd_consts_arc()); my_fd.set_rd_virtuals_arc(donor_fd.rd_virtuals_arc()); my_fd.set_rd_pendingfields_arc(donor_fd.rd_pendingfields_arc()); + // `AbstractFailDescr.rd_locs` is a backend slot upstream, written + // only by `store_info_on_descr`, which every upstream backend runs — + // so `copy_all_attributes_from` has nothing to carry there. Pyre also + // seeds it in `store_final_boxes_in_guard` with the resume numbering's + // identity-with-holes layout, which cranelift and wasm keep because + // they leave failargs in their logical slots. That seed belongs to + // the payload copied just above: the known-class bitfield inside + // `rd_numb` holds one bit per non-hole Ref livebox, and + // `deserialize_optimizer_knowledge` can only find those bits by + // masking its fail args with the same holes. + // + // Every caller hands a descr the optimizer minted for a guard put in + // place of an already-emitted one — `replace_guard_op`, the + // GUARD_VALUE and GUARD_CLASS strengthening arms of + // `optimize_guard_value` / `replace_old_guard_with_guard_class`, and + // the vectorizer's `inhert_attributes` — and such a descr never + // reaches `store_final_boxes_in_guard`, so it has no layout of its own. + my_fd.set_rd_locs(donor_fd.rd_locs().to_vec()); // compile.py — chain.clone() preserves the donor's // (already-flattened) accumulator chain on self, identity-stable. let donor_chain = donor_fd.vector_info(); diff --git a/majit/majit-metainterp/src/jitcode/assembler.rs b/majit/majit-metainterp/src/jitcode/assembler.rs index a550c820926..81160b30a8a 100644 --- a/majit/majit-metainterp/src/jitcode/assembler.rs +++ b/majit/majit-metainterp/src/jitcode/assembler.rs @@ -3542,6 +3542,64 @@ impl JitCodeBuilder { self.inline_call_r_v(sub_jitcode_idx, &[], None); } + /// Emit RPython's canonical `inline_call_{r,ir,irf}_{i,r,f,v}` + /// bytecode shape. + /// + /// This is deliberately separate from [`Self::inline_call_r_r`] and its + /// siblings. Those methods predate the codewriter port and emit pyre's + /// opaque `inline_call_nested_ext/P` payload, including explicit callee + /// destination registers. `jtransform.py handle_regular_call` instead + /// emits a JitCode descr followed by kind-separated varlists; the callee's + /// `MIFrame.setup_call` places each list at the start of its matching + /// register bank. `pyre-jit`'s SSA assembler uses this method for that + /// orthodox stream. + pub fn canonical_inline_call( + &mut self, + key: &'static str, + sub_jitcode_idx: u16, + args_i: Option<&[u16]>, + args_r: Option<&[u16]>, + args_f: Option<&[u16]>, + result: Option<(JitArgKind, u16)>, + ) { + self.write_insn(key); + self.push_u16(sub_jitcode_idx); + if let Some(args) = args_i { + self.push_canonical_inline_varlist(args, JitArgKind::Int); + } + if let Some(args) = args_r { + self.push_canonical_inline_varlist(args, JitArgKind::Ref); + } + if let Some(args) = args_f { + self.push_canonical_inline_varlist(args, JitArgKind::Float); + } + if let Some((kind, dst)) = result { + match kind { + JitArgKind::Int => self.touch_reg(dst), + JitArgKind::Ref => self.touch_ref_reg(dst), + JitArgKind::Float => self.touch_float_reg(dst), + } + self.push_reg_u8(dst, "canonical inline_call return"); + } + } + + fn push_canonical_inline_varlist(&mut self, args: &[u16], kind: JitArgKind) { + if args.len() > u8::MAX as usize { + self.encoding_overflow = true; + self.push_u8(0); + return; + } + self.push_u8(args.len() as u8); + for &src in args { + match kind { + JitArgKind::Int => self.touch_reg(src), + JitArgKind::Ref => self.touch_ref_reg(src), + JitArgKind::Float => self.touch_float_reg(src), + } + self.push_reg_u8(src, "canonical inline_call argument"); + } + } + pub fn inline_call_r_i( &mut self, sub_jitcode_idx: u16, diff --git a/majit/majit-metainterp/src/jitcode/mod.rs b/majit/majit-metainterp/src/jitcode/mod.rs index 5bb34342ae2..a38a7e43dba 100644 --- a/majit/majit-metainterp/src/jitcode/mod.rs +++ b/majit/majit-metainterp/src/jitcode/mod.rs @@ -590,6 +590,11 @@ pub struct JitCode { /// `JitCodeBuilder` populates this during runtime per-CodeObject /// emission. pub exec: JitCodeExecState, + /// Which descriptor namespace bytecode `d`/`j` operands address. + /// `from_canonical` wraps a source-translator JitCode whose operands use + /// the process-wide codewriter table; `JitCodeBuilder` creates a runtime + /// Python-code body whose operands use `exec.descrs`. + uses_global_descr_pool: bool, /// Reachable symbolic residual targets, computed after the runtime wrapper /// has received its final function-address bindings and descriptor pool. /// @@ -633,6 +638,7 @@ impl JitCode { Self { core: majit_translate::jitcode::JitCode::new(name), exec: JitCodeExecState::default(), + uses_global_descr_pool: false, reachable_symbolic_residuals: std::sync::OnceLock::new(), } } @@ -662,10 +668,15 @@ impl JitCode { jit_merge_point_offset, ..JitCodeExecState::default() }, + uses_global_descr_pool: true, reachable_symbolic_residuals: std::sync::OnceLock::new(), } } + pub fn uses_global_descr_pool(&self) -> bool { + self.uses_global_descr_pool + } + /// Borrow the canonical core (e.g. for serialization that /// re-serializes only the canonical fields). pub fn core(&self) -> &majit_translate::jitcode::JitCode { @@ -683,7 +694,9 @@ impl JitCode { impl Default for JitCode { fn default() -> Self { - Self::from_canonical(majit_translate::jitcode::JitCode::default()) + let mut jitcode = Self::from_canonical(majit_translate::jitcode::JitCode::default()); + jitcode.uses_global_descr_pool = false; + jitcode } } @@ -692,6 +705,7 @@ impl Clone for JitCode { Self { core: self.core.clone(), exec: self.exec.clone(), + uses_global_descr_pool: self.uses_global_descr_pool, reachable_symbolic_residuals: self.reachable_symbolic_residuals.clone(), } } diff --git a/majit/majit-metainterp/src/optimizeopt/virtualize.rs b/majit/majit-metainterp/src/optimizeopt/virtualize.rs index 1543be1c5f6..01c283baf40 100644 --- a/majit/majit-metainterp/src/optimizeopt/virtualize.rs +++ b/majit/majit-metainterp/src/optimizeopt/virtualize.rs @@ -1173,10 +1173,21 @@ impl OptVirtualize { field_descr.offset(), ); } - let field_val = match &info { + // virtualize.py `optimize_GETFIELD_GC_I`: + // fieldop = opinfo.getfield(op.getdescr()) + // self.make_equal_to(op, fieldop) + // `info.py AbstractStructPtrInfo.getfield` returns `self._fields[index]` + // — the stored Box itself, so `make_equal_to` forwards the read onto + // that box and a later `get_box_replacement` walks the box's own + // `_forwarded` chain. A virtualizable slot holds an `Operand`, so hand + // that operand over directly. Reading the slot back by POSITION + // instead re-keys it into whichever inputarg namespace the current + // pass owns: `VirtualizableTracker::init` seeds the slots from the + // seeding pass's inputargs, and an unrolled body owns a disjoint + // range, so the position resolves there to a fresh host that carries + // none of the facts the seeded box holds. + let field_box: Option = match &info { _ if !slot_resolvable => None, - PtrInfo::Virtual(vinfo) => get_field(&vinfo.fields, field_idx), - PtrInfo::VirtualStruct(vinfo) => get_field(&vinfo.fields, field_idx), PtrInfo::Virtualizable(vstate) => vstate .fields .iter() @@ -1190,7 +1201,18 @@ impl OptVirtualize { op.result_type(), ) }) - .map(|(_, b)| b.to_opref()), + .map(|(_, b)| b.get_box_replacement(false)), + _ => None, + }; + if let Some(b_val) = field_box { + let b_old = Operand::from_bound_op(op_rc); + ctx.make_equal_to(&b_old, &b_val); + return OptimizationResult::Remove; + } + let field_val = match &info { + _ if !slot_resolvable => None, + PtrInfo::Virtual(vinfo) => get_field(&vinfo.fields, field_idx), + PtrInfo::VirtualStruct(vinfo) => get_field(&vinfo.fields, field_idx), _ => None, }; if let Some(val_ref) = field_val { diff --git a/majit/majit-metainterp/src/pyjitpl/dispatch.rs b/majit/majit-metainterp/src/pyjitpl/dispatch.rs index 33969077b9c..40285af9564 100644 --- a/majit/majit-metainterp/src/pyjitpl/dispatch.rs +++ b/majit/majit-metainterp/src/pyjitpl/dispatch.rs @@ -9293,6 +9293,49 @@ where self.replace_box(ctx, opref, const_ref, Type::Float); } } + // pyjitpl.py opimpl_guard_class: + // clsbox = self.cls_of_box(box) + // if not self.metainterp.heapcache.is_class_known(box): + // self.metainterp.generate_guard(rop.GUARD_CLASS, box, clsbox, + // resumepc=orgpc) + // self.metainterp.heapcache.class_now_known(box) + // return clsbox + // `jtransform.py handle_getfield_typeptr` emits this for every + // read of the header's class word; the result lands in the bank + // the read was allocated to (`BC_GUARD_CLASS` int, + // `BC_GUARD_CLASS_R` ref). + byte @ (jitcode::insns::BC_GUARD_CLASS | jitcode::insns::BC_GUARD_CLASS_R) => { + let (opcode_pc, src, dst) = { + let frame = self.frames.current_mut(); + let opcode_pc = frame.code_cursor - 1; + let src = frame.next_reg() as usize; + let dst = frame.next_reg() as usize; + (opcode_pc, src, dst) + }; + let (opref, concrete) = self.read_ref_reg(src); + if concrete == 0 { + return TraceAction::Abort; + } + let typeptr = self.read_typeptr_from_exception(concrete); + let cls_const = ctx.const_int(typeptr); + if !ctx.heap_cache().is_class_known(opref) { + self.record_state_guard( + ctx, + sym, + majit_ir::OpCode::GuardClass, + &[opref, cls_const], + opcode_pc, + /* after_residual_call */ false, + ); + ctx.heap_cache_mut().class_now_known(opref, typeptr); + } + if byte == jitcode::insns::BC_GUARD_CLASS { + self.set_int_reg(dst, Some(cls_const), Some(typeptr)); + } else { + let cls_ref = ctx.const_ref(typeptr); + self.set_ref_reg(dst, Some(cls_ref), Some(typeptr)); + } + } jitcode::insns::BC_RAISE => { // pyjitpl.py opimpl_raise: // if not self.metainterp.heapcache.is_class_known(exc_value_box): diff --git a/majit/majit-trace/src/heapcache.rs b/majit/majit-trace/src/heapcache.rs index 9cb99cd78f6..aaef5b01b40 100644 --- a/majit/majit-trace/src/heapcache.rs +++ b/majit/majit-trace/src/heapcache.rs @@ -1316,7 +1316,19 @@ impl HeapCache { // aggressive arm). Pyre's `is_call()` is the broader // `_CALL_FIRST..=_CALL_LAST` range, so use the narrow // `is_plain_call()` predicate to mirror upstream's enumeration. + // `CALL_PURE_*` reaches here spelled as itself, where upstream would + // still be holding the plain `CALL_*` it recorded: + // `MIFrame.execute_varargs` (pyjitpl.py) records the residual through + // `execute_and_record_varargs(rop.CALL_*)` -- which is what runs + // `invalidate_caches` -- and only then does + // `record_result_of_call_pure` rewrite the opcode to `CALL_PURE_*`. + // So the elidable early-return below is on upstream's path for exactly + // these calls, and gating it on `is_plain_call` alone drops an + // `EF_ELIDABLE_CANNOT_RAISE` residual onto the blanket + // `reset_keep_likely_virtuals`, which bumps `head_version` and voids + // every box's class and nullity knowledge mid-trace. if opnum.is_plain_call() + || opnum.is_call_pure() || opnum.is_call_loopinvariant() || opnum.is_cond_call_value() || opnum == OpCode::CondCallN diff --git a/majit/majit-translate/src/codewriter/assembler.rs b/majit/majit-translate/src/codewriter/assembler.rs index abf61734582..18af151dc8e 100644 --- a/majit/majit-translate/src/codewriter/assembler.rs +++ b/majit/majit-translate/src/codewriter/assembler.rs @@ -1944,6 +1944,41 @@ impl Assembler { // `int_guard_value` and the subsequent `residual_call_*` // consume — backend lowering of the actual vtable slot read // is not yet implemented. + // blackhole.py `@arguments("cpu", "r", returns="i") + // bhimpl_guard_class` — one ref register in, the class out. + // The result bank follows the register the original read was + // allocated to (`guard_class/r>i` or `guard_class/r>r`), see + // `OpKind::GuardClass`. + OpKind::GuardClass { base } => { + let (reg, kc) = self.lookup_reg_with_kind_var(base, regallocs); + // The base is the object whose header word is read, so it is a + // ref register; `bhimpl_guard_class` takes `"r"` and both + // registered keys (`guard_class/r>i`, `guard_class/r>r`) spell + // it that way. An Int-banked base would form a key nothing + // registers, so fail loud at emit time. + assert_eq!( + kc, 'r', + "guard_class base must be ref-kind ('r'), got '{kc}'" + ); + state.code.push(reg); + argcodes.push(kc); + let result = op + .result + .as_ref() + .expect("guard_class must produce a result register"); + argcodes.push('>'); + let (reg, kc) = self.lookup_reg_with_kind_var(result, regallocs); + assert!( + kc == 'i' || kc == 'r', + "guard_class result must be int- or ref-kind, got '{kc}'" + ); + argcodes.push(kc); + state.code.push(reg); + let opname = op_kind_to_opname(&op.kind); + let key = format!("{opname}/{argcodes}"); + let opnum = self.get_opnum(&key); + state.code[startposition] = opnum; + } OpKind::VtableMethodPtr { receiver, trait_root, @@ -3066,6 +3101,7 @@ impl Assembler { OpKind::GuardTrue { .. } => "GuardTrue", OpKind::GuardFalse { .. } => "GuardFalse", OpKind::GuardValue { .. } => "GuardValue", + OpKind::GuardClass { .. } => "GuardClass", OpKind::VtableMethodPtr { .. } => "VtableMethodPtr", OpKind::IndirectCall { .. } => "IndirectCall", OpKind::VableFieldRead { .. } => "VableFieldRead", @@ -5171,6 +5207,8 @@ fn op_kind_to_opname(kind: &crate::model::OpKind) -> String { } OpKind::GuardTrue { .. } => "guard_true".into(), OpKind::GuardFalse { .. } => "guard_false".into(), + // jtransform.py handle_getfield_typeptr: `SpaceOperation('guard_class', [op.args[0]], op.result)`. + OpKind::GuardClass { .. } => "guard_class".into(), OpKind::GuardValue { kind_char, .. } => { // `rpython/jit/codewriter/jtransform.py:611` emits one of // `int_guard_value` / `ref_guard_value` / `float_guard_value` diff --git a/majit/majit-translate/src/codewriter/call.rs b/majit/majit-translate/src/codewriter/call.rs index 0d26fcd8fed..5a467918b7e 100644 --- a/majit/majit-translate/src/codewriter/call.rs +++ b/majit/majit-translate/src/codewriter/call.rs @@ -1217,6 +1217,9 @@ pub struct CallControl { /// Candidate targets — graphs we will inline. /// RPython: `CallControl.candidate_graphs`. candidate_graphs: HashSet, + /// `PipelineConfig::helper_graphs` — host-declared BFS seeds beside the + /// portals (`call.py inline_calls_to`). + helper_seed_graphs: Vec, /// RPython: `JitDriverStaticData` — metadata for each jitdriver. /// `jitdrivers_sd[i]` holds the green/red arg layout for driver i. @@ -1894,6 +1897,7 @@ impl CallControl { trait_method_impls: HashMap::new(), method_to_impl_types: HashMap::new(), candidate_graphs: HashSet::new(), + helper_seed_graphs: Vec::new(), jitdrivers_sd: Vec::new(), jitcodes: indexmap::IndexMap::new(), function_fnaddrs: HashMap::new(), @@ -3129,6 +3133,18 @@ impl CallControl { /// that have access to the compiled helper surface can preload the /// equivalent integer address here so `get_jitcode()` and /// `fnaddr_for_target()` no longer fall back to symbolic hashes. + /// Seed `path` into the `find_all_graphs` BFS beside the portals. + /// + /// `call.py` seeds the `inline_calls_to` helper graphs because the + /// codewriter lowers an operation straight to a residual call of the + /// helper, so no graph calls it in source. A host lowering an opcode to + /// a residual the same way names the residual's body here; the seed is a + /// candidate like a portal, and its callees join the closure under the + /// same policy as every other call. + pub fn register_helper_graph(&mut self, path: CallPath) { + self.helper_seed_graphs.push(path); + } + pub fn register_function_fnaddr(&mut self, path: CallPath, fnaddr: i64) { self.function_fnaddrs.insert(path, fnaddr); } @@ -3937,6 +3953,15 @@ impl CallControl { todo.push(path.clone()); } } + // The host's own `inline_calls_to`: the bodies behind the residual + // calls it lowers opcodes to (`register_helper_graph`). A seed with + // no graph is reported by the drain loop below. + let helper_seeds = self.helper_seed_graphs.clone(); + for path in helper_seeds { + if self.candidate_graphs.insert(path.clone()) { + todo.push(path); + } + } // call.py:59-64 — seed the BFS with builtin oopspec helpers so // `int_abs` / `int_floordiv` / `int_mod` / `ll_math.ll_math_sqrt` // are reachable even when the portal does not call them @@ -4701,6 +4726,16 @@ impl CallControl { for wrapper in wrappers { self.get_jitcode(&wrapper); } + // A helper seed (`register_helper_graph`) is called only from the + // residual the host lowers to, never from a graph being flattened, + // so no call site allocates its JitCode either; allocate it here so + // a descent can resolve the body by path. + let helper_seeds = self.helper_seed_graphs.clone(); + for path in helper_seeds { + if self.function_graphs.contains_key(&path) { + self.get_jitcode(&path); + } + } } /// RPython: `CallControl.enum_pending_graphs()` (call.py). @@ -9014,6 +9049,7 @@ fn op_can_raise(op: &OpKind) -> RaiseClass { OpKind::GuardTrue { .. } | OpKind::GuardFalse { .. } | OpKind::GuardValue { .. } + | OpKind::GuardClass { .. } | OpKind::JitDebug { .. } | OpKind::AssertGreen { .. } | OpKind::CurrentTraceLength diff --git a/majit/majit-translate/src/codewriter/format.rs b/majit/majit-translate/src/codewriter/format.rs index 66b12b294a3..ab64dc332d6 100644 --- a/majit/majit-translate/src/codewriter/format.rs +++ b/majit/majit-translate/src/codewriter/format.rs @@ -501,7 +501,7 @@ fn op_name(op: &crate::model::SpaceOperation) -> String { } => { format!( "call_elidable_{}_{result_kind}", - kind_signature(args_i, args_r, args_f) + kind_signature(args_i, args_r, args_f, *result_kind) ) } OpKind::CallResidual { @@ -513,7 +513,7 @@ fn op_name(op: &crate::model::SpaceOperation) -> String { } => { format!( "residual_call_{}_{result_kind}", - kind_signature(args_i, args_r, args_f) + kind_signature(args_i, args_r, args_f, *result_kind) ) } OpKind::CallMayForce { @@ -525,7 +525,7 @@ fn op_name(op: &crate::model::SpaceOperation) -> String { } => { format!( "call_may_force_{}_{result_kind}", - kind_signature(args_i, args_r, args_f) + kind_signature(args_i, args_r, args_f, *result_kind) ) } OpKind::InlineCall { @@ -537,12 +537,15 @@ fn op_name(op: &crate::model::SpaceOperation) -> String { } => { format!( "inline_call_{}_{result_kind}", - kind_signature(args_i, args_r, args_f) + kind_signature(args_i, args_r, args_f, *result_kind) ) } OpKind::RecursiveCall { result_kind, .. } => { format!("recursive_call_{result_kind}") } + // The Debug fallback below would spell it "guardclass"; the + // canonical opname carries the underscore. + OpKind::GuardClass { .. } => "guard_class".to_string(), // For the rest, fall back on a stable Debug-derived discriminant. other => format!("{:?}", other) .split('{') @@ -556,23 +559,26 @@ fn op_name(op: &crate::model::SpaceOperation) -> String { } } -/// jtransform.py:414-435 — call-family opcode kind suffix. +/// `jtransform.py rewrite_call` — call-family opcode kind suffix. /// -/// Encodes the (int, ref, float) arg tuple as a single-character -/// signature ("i", "r", "f", "ir", "irf", …). Empty bins drop out so -/// `(args_i=[a], args_r=[], args_f=[])` produces `"i"`. -fn kind_signature(args_i: &[T], args_r: &[T], args_f: &[T]) -> String { - let mut out = String::new(); - if !args_i.is_empty() { - out.push('i'); - } - if !args_r.is_empty() { - out.push('r'); - } - if !args_f.is_empty() { - out.push('f'); +/// One of exactly three signatures, chosen by the widest bin in play rather +/// than by which bins are occupied: `"irf"` when a float appears among the +/// arguments or as the result, else `"ir"` when an int does, else `"r"`. So +/// `(args_i=[a], args_r=[], args_f=[])` is `"ir"` and an all-empty call is +/// `"r"` — the `r` bin is named even when it is empty, because the sublists +/// the signature announces are positional and the reader counts them. +/// +/// The result kind is a signature input, not just the suffix after it: a +/// float-returning call with no float argument is still `"irf"`. +fn kind_signature(args_i: &[T], args_r: &[T], args_f: &[T], result_kind: char) -> &'static str { + if !args_f.is_empty() || result_kind == 'f' { + "irf" + } else if !args_i.is_empty() { + "ir" + } else { + let _ = args_r; + "r" } - out } fn op_args_repr(op: &crate::model::SpaceOperation) -> String { @@ -627,6 +633,7 @@ fn op_args_repr(op: &crate::model::SpaceOperation) -> String { args_i, args_r, args_f, + result_kind, .. } | OpKind::CallResidual { @@ -635,6 +642,7 @@ fn op_args_repr(op: &crate::model::SpaceOperation) -> String { args_i, args_r, args_f, + result_kind, .. } | OpKind::CallMayForce { @@ -643,18 +651,22 @@ fn op_args_repr(op: &crate::model::SpaceOperation) -> String { args_i, args_r, args_f, + result_kind, .. } => { let mut parts = vec![call_funcptr_repr(funcptr)]; - // jtransform.py:430-433 — emit each ListOfKind only when the - // matching kind char is in the signature. - if !args_i.is_empty() { + // jtransform.py `rewrite_call` — emit each ListOfKind exactly when + // the matching kind char is in the signature, which is not the + // same test as "the bin is non-empty": `ir` over no int arguments + // still announces an empty `I[]`. + let kinds = kind_signature(args_i, args_r, args_f, *result_kind); + if kinds.contains('i') { parts.push(list_of_kind_repr_vars('i', args_i)); } - if !args_r.is_empty() { + if kinds.contains('r') { parts.push(list_of_kind_repr_vars('r', args_r)); } - if !args_f.is_empty() { + if kinds.contains('f') { parts.push(list_of_kind_repr_vars('f', args_f)); } // jtransform.py:434 — descr is the last sublist when set. @@ -671,6 +683,7 @@ fn op_args_repr(op: &crate::model::SpaceOperation) -> String { args_i, args_r, args_f, + result_kind, .. } => { let head = match jitcode.try_index() { @@ -678,13 +691,14 @@ fn op_args_repr(op: &crate::model::SpaceOperation) -> String { None => format!("", jitcode.name), }; let mut parts = vec![head]; - if !args_i.is_empty() { + let kinds = kind_signature(args_i, args_r, args_f, *result_kind); + if kinds.contains('i') { parts.push(list_of_kind_repr_vars('i', args_i)); } - if !args_r.is_empty() { + if kinds.contains('r') { parts.push(list_of_kind_repr_vars('r', args_r)); } - if !args_f.is_empty() { + if kinds.contains('f') { parts.push(list_of_kind_repr_vars('f', args_f)); } out.push_str(&parts.join(", ")); @@ -815,6 +829,11 @@ fn op_result_kind(kind: &crate::model::OpKind) -> RegKind { OpKind::IsConstant { .. } | OpKind::IsVirtual { .. } | OpKind::VableArrayLen { .. } => { RegKind::Int } + // The class word of the header, in whichever bank the read it + // replaced was allocated to (`OpKind::GuardClass`); the formatter + // has no regalloc to ask, and the int bank is where the class + // pointer hints (`record_exact_class/ri`) already carry it. + OpKind::GuardClass { .. } => RegKind::Int, // Result-less or pyre-only debug variants — `op_args_repr` // only reaches this fall-through when `op.result.is_some()`, // so any miss surfaces as a real coverage gap to extend. diff --git a/majit/majit-translate/src/codewriter/insns.rs b/majit/majit-translate/src/codewriter/insns.rs index a22c48e983d..4519e3540c2 100644 --- a/majit/majit-translate/src/codewriter/insns.rs +++ b/majit/majit-translate/src/codewriter/insns.rs @@ -607,6 +607,15 @@ pub const BC_ASSERT_NOT_NONE: u8 = 200; // `blackhole.py @arguments("r", "i")`. pub const BC_RECORD_EXACT_CLASS: u8 = 201; +// `guard_class/r>i` and `guard_class/r>r` — RPython `blackhole.py` +// `bhimpl_guard_class(cpu, struct): return cpu.bh_classof(struct)`, emitted +// by `jtransform.py handle_getfield_typeptr` for every read of the +// object header's class word. Upstream's result is `Ptr(OBJECT_VTABLE)`; +// pyre's `PyObject.ob_type` is read as a raw word in some graphs and as a +// GC ref in others, and the guard keeps the bank of the read it replaced. +pub const BC_GUARD_CLASS: u8 = 237; +pub const BC_GUARD_CLASS_R: u8 = 238; + pub const MAX_HOST_CALL_ARITY: usize = 16; /// Lookup a bytecode opcode by its `opname/argcodes` key. @@ -1080,6 +1089,9 @@ pub fn wellknown_bh_insns() -> IndexMap<&'static str, u8> { // constant int class pointer as a ConstPtr before recording. m.insert("assert_not_none/r", BC_ASSERT_NOT_NONE); m.insert("record_exact_class/ri", BC_RECORD_EXACT_CLASS); + // `guard_class` — `blackhole.py bhimpl_guard_class`; see `BC_GUARD_CLASS`. + m.insert("guard_class/r>i", BC_GUARD_CLASS); + m.insert("guard_class/r>r", BC_GUARD_CLASS_R); // Float comparisons — `blackhole.py:721-746` // `bhimpl_float_{lt,le,eq,ne,gt,ge}` — float pair → int (0/1). diff --git a/majit/majit-translate/src/codewriter/jitcode.rs b/majit/majit-translate/src/codewriter/jitcode.rs index 110f9c4584d..ba91309c92e 100644 --- a/majit/majit-translate/src/codewriter/jitcode.rs +++ b/majit/majit-translate/src/codewriter/jitcode.rs @@ -344,6 +344,12 @@ pub struct DescentBlockerSummary { /// scan never enters reports no blocker — so this cannot be expressed as /// one of the two above and declines on its own. pub body_not_walked: bool, + /// The byte position of the first op of this body that made some path + /// effectful — a residual call, a heap write, or an `inline_call` into a + /// callee that may execute an effect (a cycle or an unread body answers + /// so too). Diagnostic only: it names what turned a body's blockers into + /// declines. + pub first_effect_pc: Option, } /// Answers computed on demand from an assembled [`JitCode`] body. @@ -356,8 +362,19 @@ pub struct DerivedBodyFacts { /// by the symbolic hash standing in for its funcbox. Read through /// [`JitCode::descent_blocker_summary`]. descent_blocker_summary: OnceLock, + /// The same answer computed with the argument-array length a call site + /// knows, indexed by that length. Read through + /// [`JitCode::descent_blocker_summary_for_entry_len`]. + descent_blocker_summary_by_entry_len: + [OnceLock; DESCENT_ENTRY_LEN_SLOTS], } +/// How many distinct entry argument-array lengths +/// [`JitCode::descent_blocker_summary_for_entry_len`] caches. A generated +/// gateway is called with the arity its signature names, so the lengths a body +/// is ever asked about form a short prefix; a longer one recomputes. +pub const DESCENT_ENTRY_LEN_SLOTS: usize = 8; + mod oncelock_usize_serde { use std::sync::OnceLock; @@ -472,6 +489,40 @@ impl JitCode { *self.derived.descent_blocker_summary.get_or_init(compute) } + /// The already-computed answer of [`Self::descent_blocker_summary`], or + /// `None` when nothing has asked for it yet. + /// + /// A caller that must decide whether its answer may be stored reads the + /// slot without filling it: `get_or_init` would take a closure it is not + /// yet entitled to commit. + pub fn descent_blocker_summary_if_computed(&self) -> Option { + self.derived.descent_blocker_summary.get().copied() + } + + /// The same answer as [`Self::descent_blocker_summary`], for a caller that + /// knows the length of the argument array the body is entered with. + /// + /// The scan reads the assembled body and the build-time descr / jitcode + /// tables, so its answer is a function of the body and that length alone + /// and one slot per length is a complete cache. Without it the gate pays a + /// whole-body worklist dataflow — which recurses into callee bodies + /// uncached, so a shared callee is re-walked once per path — on every call + /// site, every walk. + pub fn descent_blocker_summary_for_entry_len( + &self, + entry_len: usize, + compute: impl FnOnce() -> DescentBlockerSummary, + ) -> DescentBlockerSummary { + match self + .derived + .descent_blocker_summary_by_entry_len + .get(entry_len) + { + Some(slot) => *slot.get_or_init(compute), + None => compute(), + } + } + /// Commit the body once assembly has produced it. Panics on second /// call (RPython equivalent: `JitCode.setup` is also called once per /// jitcode lifetime). diff --git a/majit/majit-translate/src/codewriter/jtransform.rs b/majit/majit-translate/src/codewriter/jtransform.rs index 3dc58c111d8..37281f6d61a 100644 --- a/majit/majit-translate/src/codewriter/jtransform.rs +++ b/majit/majit-translate/src/codewriter/jtransform.rs @@ -507,6 +507,19 @@ enum RewriteResult { Keep, } +/// `jtransform.py is_typeptr_getset`: the access names the class +/// word of the object header. Upstream keys on the field name `typeptr` +/// and the struct's `typeptr` hint; pyre's header is `PyObject { ob_type, +/// w_class }`, and only the `ob_type` word is the class the tracer guards on +/// — `w_class` is the Python-level class, an ordinary field to a guard. +fn is_typeptr_field(field: &FieldDescriptor) -> bool { + let owner_leaf = field + .owner_root + .as_deref() + .map(|owner| owner.rsplit("::").next().unwrap_or(owner)); + field.name == "ob_type" && owner_leaf == Some("PyObject") +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ResolvedCallResult { kind: char, @@ -589,6 +602,22 @@ pub(crate) fn jit_marker_key_from_target( } } +/// The false result `can_enter_jit` leaves behind (`rewrite_can_enter_jits` +/// runs after `make_jitcodes`, so the portal jitcode never enters through it). +/// Upstream's result is `Constant(False)`, a Bool; stamping the same +/// concretetype here lets the `bool` hop `set_branch` wrapped around the +/// condition rewrite to an identity instead of `int_is_true`, so the +/// constant exitswitch fold after rewriting still sees the constant. +fn can_enter_jit_false_result(result: &crate::flowspace::model::Variable) -> SpaceOperation { + result.set_concretetype(Some( + crate::translator::rtyper::lltypesystem::lltype::LowLevelType::Bool, + )); + SpaceOperation { + result: Some(result.clone()), + kind: OpKind::ConstInt(0), + } +} + /// Split a run of [`Variable`]s into (ints, refs, floats) per upstream /// `make_three_lists` (`jtransform.py`). Void values are /// dropped, matching the upstream filter; Unknown defaults to `Ref`. @@ -624,6 +653,67 @@ fn split_args_by_kind( (ints, refs, floats) } +/// `jtransform.py _rewrite_cmp_ptrs` (`rewrite_op_ptr_eq` / +/// `rewrite_op_ptr_ne`): a pointer compared against NULL is the unary +/// `ptr_iszero` / `ptr_nonzero` test over the other operand, not a +/// two-operand `ptr_eq`. Pyre's front materialises NULL as a Variable +/// defined by [`OpKind::ConstRefNull`], so the constant is found through its +/// definition. The unary form is what `optimize_goto_if_not` fuses into +/// `goto_if_not_ptr_iszero`, which the walker answers from the heap cache +/// without recording once the nullity is known; `ptr_eq(x, NULL)` fused into +/// `goto_if_not_ptr_eq` still recorded `ptr_eq` + `guard_false` per test. +fn null_test_rewrite( + graph: &FunctionGraph, + op: &SpaceOperation, + eq: bool, + lhs: &crate::flowspace::model::Variable, + rhs: &crate::flowspace::model::Variable, +) -> Option { + // The block under rewrite still holds its original operations, so a NULL + // defined there is the `ptr::null[_mut]()` / `PY_NULL` call the + // `rtype_ptr_null` arm of `rewrite_op_direct_call` has not yet folded. + let is_null_const = |variable: &crate::flowspace::model::Variable| { + graph.blocks.iter().any(|block| { + block.operations.iter().any(|def| { + def.result.as_ref() == Some(variable) + && match &def.kind { + OpKind::ConstRefNull => true, + OpKind::Call { + target: CallTarget::FunctionPath { segments }, + args, + result_ty, + } => { + args.is_empty() + && matches!(result_ty, ValueType::Ref(_)) + && resolves_to_null_ptr_builtin(segments) + } + _ => false, + } + }) + }) + }; + let operand = if is_null_const(rhs) { + lhs + } else if is_null_const(lhs) { + rhs + } else { + return None; + }; + if let Some(result) = &op.result { + result.set_concretetype(Some( + crate::translator::rtyper::lltypesystem::lltype::LowLevelType::Bool, + )); + } + Some(RewriteResult::Replace(vec![SpaceOperation { + result: op.result.clone(), + kind: OpKind::UnaryOp { + op: if eq { "ptr_iszero" } else { "ptr_nonzero" }.into(), + operand: operand.clone(), + result_ty: ValueType::Int, + }, + }])) +} + /// Pyre's frontend materialises source constants as SSA Variables. Recover /// the upstream `Constant`/`Variable` distinction at the graph boundary. fn is_source_constant_variable( @@ -1007,6 +1097,7 @@ impl<'a> Transformer<'a> { /// directly. pub fn transform(&mut self, graph: &FunctionGraph) -> GraphTransformResult { let mut rewritten = graph.clone(); + join_blocks(&mut rewritten); // `jtransform.py transform_graph` opens with // `constant_fold_ll_issubclass(graph, cpu)`, which folds a @@ -2061,7 +2152,12 @@ impl<'a> Transformer<'a> { // because Rust's `==`/`!=` is one AST node regardless of operand type; // the jtransform layer is where RPython branches on operand kind. // Both operands Ref → rewrite to `ptr_eq`/`ptr_ne`. Mixed/Int operands - // stay as `int_eq`/`int_ne`. + // stay as `int_eq`/`int_ne`. `rptr.py:167-184 + // pairtype(PtrRepr, Repr).rtype_eq/ne` calls + // `hop.inputargs(r_ptr, r_ptr)` — both operands are already + // ptr-typed here, so there is no cast — and pyre's blackhole wires + // `bhimpl_ptr_eq` / `bhimpl_ptr_ne` at `bh_binop_r_to_i`, so the + // `ptr_eq/rr>i` opname dispatches without `cast_ptr_to_int`. OpKind::BinOp { op: binop_name, lhs, @@ -2071,6 +2167,19 @@ impl<'a> Transformer<'a> { && self.get_value_kind_var(lhs) == 'r' && self.get_value_kind_var(rhs) == 'r' => { + if let Some(rewritten) = null_test_rewrite(graph, &op, binop_name == "eq", lhs, rhs) + { + return rewritten; + } + // The comparison answers an integer, so the result banks as + // one; without the stamp it keeps the `'r'` kind the unified + // `BinOp` carried in and `op_kind_to_opname_with_kinds` spells + // an opname no blackhole handler registers. + self.stamp_value_kind( + graph, + op.result.clone(), + crate::codewriter::type_state::ConcreteType::Signed, + ); let new_op = if binop_name == "eq" { "ptr_eq" } else { @@ -2228,44 +2337,6 @@ impl<'a> Transformer<'a> { // for the missing rtyper cast remains the canonical // convergence path; this jtransform recovery is the // bridge until that lands. - // eq/ne with BOTH operands ref-kind → emit ptr_eq / ptr_ne - // directly. PyPy `rpython/rtyper/rptr.py:167-184 - // pairtype(PtrRepr, Repr).rtype_eq/ne` calls - // `hop.inputargs(r_ptr, r_ptr)` (both already ptr-typed in - // this branch — no cast) and emits `ptr_eq` / `ptr_ne`. - // Pyre's blackhole has `bhimpl_ptr_eq` / `bhimpl_ptr_ne` - // wired at `bh_binop_r_to_i`, so the resulting - // `ptr_eq/rr>i` opname dispatches without going through - // `cast_ptr_to_int`. - OpKind::BinOp { - op: binop_name, - lhs, - rhs, - result_ty, - } if matches!(binop_name.as_str(), "eq" | "ne") - && self.get_value_kind_var(lhs) == 'r' - && self.get_value_kind_var(rhs) == 'r' => - { - self.stamp_value_kind( - graph, - op.result.clone(), - crate::codewriter::type_state::ConcreteType::Signed, - ); - let ptr_op = if binop_name == "eq" { - "ptr_eq" - } else { - "ptr_ne" - }; - RewriteResult::Replace(vec![SpaceOperation { - result: op.result.clone(), - kind: OpKind::BinOp { - op: ptr_op.into(), - lhs: lhs.clone(), - rhs: rhs.clone(), - result_ty: result_ty.clone(), - }, - }]) - } // Mixed-kind eq/ne (one ref + one int) or any ordered // ref-cmp (lt/le/gt/ge with a ref operand) — PRE-EXISTING // ADAPTATION: pyre's frontend admits source patterns @@ -2616,6 +2687,54 @@ impl<'a> Transformer<'a> { // the canonical `float_ne` opname here rather than // leaving an intermediate op for the float-comparison // arm in `rewrite_operation`. + // `rtype_bool` per repr for the un-rtyped `bool` hop that + // `FunctionGraph::set_branch` puts before every exitswitch: + // `BoolRepr` is the identity, `IntegerRepr.rtype_bool` is + // `int_is_true` (`rint.py`), a nullable `PtrRepr` is + // `ptr_nonzero` (`rmodel.py`). Naming the op here, ahead + // of `optimize_goto_if_not`, is half of what lets the fusion see + // it: that gate matches opnames, and `bool` is not one of them -- + // but it reads the exitswitch variable's `concretetype` FIRST, so + // the result also has to carry the `lltype.Bool` those two + // opnames return, which is why the rewrite stamps it exactly as + // `null_test_rewrite` does for the same pair. The identity arm + // is what fuses an `is_null` test -- its `ptr_iszero` result is + // already Bool -- into `goto_if_not_ptr_iszero`, which the walker + // answers from the heap cache without recording once the nullity + // is known. Left as `bool`, every null test of a traced operand + // cost `ptr_eq` + `int_is_true` + `guard_false`: six of them per + // `int + int` descent. + OpKind::UnaryOp { + op: unop_name, + operand, + .. + } if unop_name == "bool" && self.get_value_kind_var(operand) != 'f' => { + use crate::translator::rtyper::lltypesystem::lltype::LowLevelType; + if operand.concretetype() == Some(LowLevelType::Bool) { + RewriteResult::Identity(operand.clone()) + } else { + let opname = if self.get_value_kind_var(operand) == 'r' { + "ptr_nonzero" + } else { + "int_is_true" + }; + // `getkind(Bool)` is `Signed`, so the value-kind channel + // keeps banking the result as an int; the stamp only tells + // `optimize_goto_if_not` that this is the Bool its gate + // asks for. + if let Some(result) = &op.result { + result.set_concretetype(Some(LowLevelType::Bool)); + } + RewriteResult::Replace(vec![SpaceOperation { + result: op.result.clone(), + kind: OpKind::UnaryOp { + op: opname.into(), + operand: operand.clone(), + result_ty: ValueType::Int, + }, + }]) + } + } OpKind::UnaryOp { op: unop_name, operand, @@ -3406,6 +3525,32 @@ impl<'a> Transformer<'a> { ty: &ValueType, graph_name: &str, ) -> RewriteResult { + // jtransform.py `if self.is_typeptr_getset(op): return + // self.handle_getfield_typeptr(op)` — checked before anything else, + // as upstream does. Upstream also folds a Constant receiver to its + // class constant; pyre's receivers here are SSA variables (a prebuilt + // constant reaches a field read through `ConstRef`, not as an + // operand), so that arm has no input and is not spelled. + if let OpKind::FieldRead { base, .. } = &op.kind + && is_typeptr_field(field) + { + self.notes.push(GraphTransformNote { + function: graph_name.to_string(), + detail: format!("rewrite: getfield({}) → guard_class", field.name), + }); + // `jtransform.py handle_getfield_typeptr`: `-live-` first, the guard is a + // resume point. + return RewriteResult::Replace(vec![ + SpaceOperation { + result: None, + kind: OpKind::Live, + }, + SpaceOperation { + result: op.result.clone(), + kind: OpKind::GuardClass { base: base.clone() }, + }, + ]); + } // `rewrite_op_getsubstruct` applies only to an address-producing // projection. A by-value Rust field can itself have an inline-struct // layout (notably a `#[repr(transparent)]` newtype) while the operation @@ -4451,6 +4596,13 @@ impl<'a> Transformer<'a> { && matches!(receiver_root.as_str(), "mut_ptr" | "const_ptr") && args.len() == 1 { + // `ptr_iszero` produces `lltype.Bool`; the Bool is what lets + // `optimize_goto_if_not` fuse the test into the exitswitch. + if let Some(result) = &op.result { + result.set_concretetype(Some( + crate::translator::rtyper::lltypesystem::lltype::LowLevelType::Bool, + )); + } return RewriteResult::Replace(vec![SpaceOperation { result: op.result.clone(), kind: OpKind::UnaryOp { @@ -4475,6 +4627,12 @@ impl<'a> Transformer<'a> { && segments[2] == "eq" && args.len() == 2 { + // `ptr_eq` produces `lltype.Bool`, as `ptr_iszero` above. + if let Some(result) = &op.result { + result.set_concretetype(Some( + crate::translator::rtyper::lltypesystem::lltype::LowLevelType::Bool, + )); + } return RewriteResult::Replace(vec![SpaceOperation { result: op.result.clone(), kind: OpKind::BinOp { @@ -6463,10 +6621,9 @@ impl<'a> Transformer<'a> { && !jd.active { return Some(match (key, result) { - (JitMarkerKey::CanEnterJit, Some(result)) => vec![SpaceOperation { - result: Some(result.clone()), - kind: OpKind::ConstInt(0), - }], + (JitMarkerKey::CanEnterJit, Some(result)) => { + vec![can_enter_jit_false_result(result)] + } _ => Vec::new(), }); } @@ -6500,10 +6657,7 @@ impl<'a> Transformer<'a> { if key == JitMarkerKey::CanEnterJit && let Some(result) = result { - ops.push(SpaceOperation { - result: Some(result.clone()), - kind: OpKind::ConstInt(0), - }); + ops.push(can_enter_jit_false_result(result)); } Some(ops) } @@ -7568,6 +7722,112 @@ fn optimize_goto_if_not(graph: &mut FunctionGraph, block_idx: usize) -> bool { false } +/// `simplify.py join_blocks(graph)`: a link that is the single exit of a +/// block without an exitswitch and the single entry of its target is +/// deleted, the target's operations (inputargs renamed to the link's +/// arguments) appended to the block, and the target's exitswitch and exits +/// taken over. +/// +/// Upstream runs it on every flow graph before annotation, so the graphs +/// jtransform sees have no such links. Pyre's MIR front lowers every call +/// as its own block terminator, which leaves the result of a call in one +/// block and the `bool` exitswitch that tests it in the next -- across the +/// link, `optimize_goto_if_not` cannot fuse the test: every `is_null()` +/// (`ptr_iszero`) opening `py_type_check` / +/// `is_exact_builtin_instance` stayed a standalone op plus a +/// `goto_if_not`, and a descended `int + int` recorded six of them as +/// `ptr_eq` + `guard_false` pairs. Joining here, ahead of the per-block +/// rewrite, is what lets the fusion see them. +/// +/// A link carrying a constant argument is left alone: the model's +/// operations read variables only, so renaming an inputarg to a constant +/// has no operand to write it into. The joined-away block is emptied in +/// place (no operations, no exits) rather than removed, so every block +/// index -- `returnblock`, `exceptblock`, the per-block rewrite loop -- +/// keeps its meaning. +fn join_blocks(graph: &mut FunctionGraph) { + use crate::model::LinkArg; + let mut entry_count = vec![0usize; graph.blocks.len()]; + for block in &graph.blocks { + for link in &block.exits { + entry_count[link.target.0] += 1; + } + } + // `mkentrymap` seeds a synthetic entry link into `graph.startblock`. + // Counting it keeps a start block whose only in-edge is a backedge from + // reading as single-entry; `joinable` would otherwise absorb the graph's + // entry into its own predecessor and empty it. + entry_count[graph.startblock.0] += 1; + let mut seen = vec![false; graph.blocks.len()]; + seen[graph.startblock.0] = true; + let mut stack: Vec<(usize, usize)> = (0..graph.blocks[graph.startblock.0].exits.len()) + .map(|exit_idx| (graph.startblock.0, exit_idx)) + .collect(); + while let Some((prev, exit_idx)) = stack.pop() { + let Some(link) = graph.blocks[prev].exits.get(exit_idx) else { + continue; + }; + let target = link.target.0; + let joinable = graph.blocks[prev].exitswitch.is_none() + && entry_count[target] == 1 + && !graph.blocks[target].exits.is_empty() + && target != prev + && link.args.iter().all(|arg| matches!(arg, LinkArg::Value(_))); + if joinable { + debug_assert_eq!(graph.blocks[prev].exits.len(), 1); + let renaming: std::collections::HashMap< + crate::flowspace::model::Variable, + crate::flowspace::model::Variable, + > = + graph.blocks[target] + .inputargs + .iter() + .cloned() + .zip(graph.blocks[prev].exits[exit_idx].args.iter().filter_map( + |arg| match arg { + LinkArg::Value(var) => Some(var.clone()), + LinkArg::Const(_) => None, + }, + )) + .collect(); + let target_block = &graph.blocks[target]; + let moved_ops: Vec = target_block + .operations + .iter() + .map(|op| remap_op(op, &renaming)) + .collect(); + let (exitswitch, mut exits) = crate::model::remap_control_flow_metadata_var( + &target_block.exitswitch, + &target_block.exits, + |var| remap_value(var, &renaming), + |b| b, + ); + for exit in &mut exits { + exit.prevblock = Some(crate::model::BlockId(prev)); + } + { + let target_block = &mut graph.blocks[target]; + target_block.inputargs.clear(); + target_block.operations.clear(); + target_block.exitswitch = None; + target_block.exits.clear(); + } + let prev_block = &mut graph.blocks[prev]; + prev_block.operations.extend(moved_ops); + prev_block.exitswitch = exitswitch; + prev_block.exits = exits; + // Re-examine the block with its new exits, as upstream re-pushes + // `link.prevblock.exits`. + stack.extend((0..graph.blocks[prev].exits.len()).map(|idx| (prev, idx))); + continue; + } + if !seen[target] { + seen[target] = true; + stack.extend((0..graph.blocks[target].exits.len()).map(|idx| (target, idx))); + } + } +} + /// `jtransform.py:206-209` supported-opname gate for /// [`optimize_goto_if_not`]. Returns the RPython opname and the op's /// operand Variables (`tuple(op.args)`) when the op is one of the @@ -8407,6 +8667,9 @@ fn remap_op( value: remap_value(value, aliases), kind_char: *kind_char, }, + OpKind::GuardClass { base } => OpKind::GuardClass { + base: remap_value(base, aliases), + }, OpKind::IsVirtual { value, kind_char } => OpKind::IsVirtual { value: remap_value(value, aliases), kind_char: *kind_char, @@ -9621,6 +9884,175 @@ mod tests { ); } + /// A null test whose `ptr_iszero` sits in the block after a call + /// terminator: `join_blocks` folds the single-entry successor into its + /// predecessor, renaming the inputarg to the link variable, so + /// `optimize_goto_if_not` then fuses the test into the exitswitch. + #[test] + fn join_blocks_lets_goto_if_not_fuse_a_null_test_across_a_link() { + use crate::model::{ExitCase, ExitSwitch, Link}; + use crate::translator::rtyper::lltypesystem::lltype::LowLevelType; + + let mut graph = FunctionGraph::new("join_then_fuse"); + let start = graph.startblock; + let a = graph.alloc_value_var(); + let x = graph + .push_op_var( + start, + OpKind::UnaryOp { + op: "same_as".to_string(), + operand: a.clone(), + result_ty: ValueType::Ref(None), + }, + true, + ) + .unwrap(); + // `x` crosses a plain link into a block whose only work is the test. + let tester = graph.create_block(); + let x_in = graph.alloc_value_var(); + graph.push_inputarg_var(tester, x_in.clone()); + let t = graph + .push_op_var( + tester, + OpKind::UnaryOp { + op: "ptr_iszero".to_string(), + operand: x_in.clone(), + result_ty: ValueType::Int, + }, + true, + ) + .unwrap(); + t.set_concretetype(Some(LowLevelType::Bool)); + let if_false = graph.create_block(); + let if_true = graph.create_block(); + graph.set_control_flow_metadata( + tester, + Some(ExitSwitch::Value(t.clone())), + vec![ + Link::new_mixed(vec![], if_false, Some(ExitCase::Bool(false))), + Link::new_mixed(vec![], if_true, Some(ExitCase::Bool(true))), + ], + ); + graph.set_control_flow_metadata( + start, + None, + vec![Link::new_mixed( + vec![LinkArg::Value(x.clone())], + tester, + None, + )], + ); + + super::join_blocks(&mut graph); + let joined = &graph.blocks[start.0]; + assert_eq!( + joined.exits.len(), + 2, + "the tester's exits move to the start block" + ); + assert!( + graph.blocks[tester.0].operations.is_empty() && graph.blocks[tester.0].exits.is_empty(), + "the joined-away block is emptied in place" + ); + assert!( + joined.operations.iter().any(|op| matches!( + &op.kind, + OpKind::UnaryOp { op, operand, .. } if op == "ptr_iszero" && *operand == x + )), + "the moved test reads the link variable, not the dead inputarg" + ); + + assert!(super::optimize_goto_if_not(&mut graph, start.0)); + match &graph.blocks[start.0].exitswitch { + Some(ExitSwitch::Fused { opname, args }) => { + assert_eq!(opname, "ptr_iszero"); + assert_eq!(args, &vec![x]); + } + other => panic!("expected Fused exitswitch, got {other:?}"), + } + } + + /// The `bool` hop `set_branch` puts before every exitswitch has to come out + /// of the rewrite fusable. `optimize_goto_if_not` reads the exitswitch + /// variable's `concretetype` before it reads any opname, so naming the op + /// `int_is_true` is not on its own enough — nothing in the value-kind + /// channel can produce `Bool` (`concrete_to_canonical_lltype` has no such + /// case), so the rewrite is the only place that stamp can come from. + #[test] + fn transform_graph_leaves_the_bool_hop_fusable() { + use crate::model::ExitSwitch; + + let mut graph = FunctionGraph::new("bool_hop_fusable"); + let start = graph.startblock; + let a = graph + .push_op_var( + start, + OpKind::Input { + name: "a".into(), + ty: ValueType::Int, + class_root: None, + }, + true, + ) + .unwrap(); + FunctionGraph::set_concretetype_of_inline(&a, ConcreteType::Signed); + let if_true = graph.create_block(); + let if_false = graph.create_block(); + graph.set_return(if_true, None); + graph.set_return(if_false, None); + graph.set_branch(start, a.clone(), if_true, vec![], if_false, vec![]); + + let config = GraphTransformConfig::default(); + let transformed = Transformer::new(&config).transform(&graph); + match &transformed.graph.blocks[start.0].exitswitch { + Some(ExitSwitch::Fused { opname, args }) => { + assert_eq!(opname, "int_is_true"); + assert_eq!(args, &vec![a]); + } + other => panic!("expected a fused int_is_true exitswitch, got {other:?}"), + } + } + + /// The Ref-kind sibling of [`transform_graph_leaves_the_bool_hop_fusable`]. + /// A `bool` hop over a Ref operand rewrites to `ptr_nonzero`, which + /// `optimize_goto_if_not` also fuses, and it reaches that gate through the + /// same `Bool` stamp — the value-kind channel banks the result as an int + /// either way. + #[test] + fn transform_graph_leaves_the_ref_bool_hop_fusable() { + use crate::model::ExitSwitch; + + let mut graph = FunctionGraph::new("ref_bool_hop_fusable"); + let start = graph.startblock; + let a = graph + .push_op_var( + start, + OpKind::Input { + name: "a".into(), + ty: ValueType::Ref(None), + class_root: None, + }, + true, + ) + .unwrap(); + FunctionGraph::set_concretetype_of_inline(&a, ConcreteType::GcRef); + let if_true = graph.create_block(); + let if_false = graph.create_block(); + graph.set_return(if_true, None); + graph.set_return(if_false, None); + graph.set_branch(start, a.clone(), if_true, vec![], if_false, vec![]); + + let config = GraphTransformConfig::default(); + let transformed = Transformer::new(&config).transform(&graph); + match &transformed.graph.blocks[start.0].exitswitch { + Some(ExitSwitch::Fused { opname, args }) => { + assert_eq!(opname, "ptr_nonzero"); + assert_eq!(args, &vec![a]); + } + other => panic!("expected a fused ptr_nonzero exitswitch, got {other:?}"), + } + } + /// the GotoIfNotOp lowering Stage 1: a non-supported result op (`int_add`) is NOT /// fusable — `optimize_goto_if_not` returns false and leaves the /// block untouched (`jtransform.py:206-209` opname gate). diff --git a/majit/majit-translate/src/codewriter/policy.rs b/majit/majit-translate/src/codewriter/policy.rs index 2512f2016cf..aa5e14d3e05 100644 --- a/majit/majit-translate/src/codewriter/policy.rs +++ b/majit/majit-translate/src/codewriter/policy.rs @@ -567,6 +567,7 @@ pub fn collect_declared_value_types<'a>(kind: &'a OpKind, out: &mut Vec<&'a Valu | OpKind::GuardTrue { .. } | OpKind::GuardFalse { .. } | OpKind::GuardValue { .. } + | OpKind::GuardClass { .. } | OpKind::VtableMethodPtr { .. } | OpKind::VableForce { .. } | OpKind::Hint { .. } diff --git a/majit/majit-translate/src/front/mir.rs b/majit/majit-translate/src/front/mir.rs index cd628b6fe7b..be692eafd84 100644 --- a/majit/majit-translate/src/front/mir.rs +++ b/majit/majit-translate/src/front/mir.rs @@ -10780,20 +10780,31 @@ impl<'a> Lowering<'a> { self.graph.set_goto(bb_id, target_bb, link_args); return Ok(()); } - // Rust spells `&str != &str` through - // `core::cmp::impls::::ne`; RPython's - // `AbstractStringRepr.rtype_ne` lowers the same operation to - // `ll_streq` followed by `bool_not`. The flow graph's string - // `BinOp("ne")` is that decomposition's canonical spelling. - // Gate both operands on the builtin `str` pointee because the - // generic core path is shared by non-string reference impls. - // This is the comparison in PyPy's `W_Super.getattribute` - // (`name != '__class__'). + // `&a == &b` / `&a != &b` on two string-family references + // (`&Wtf8`, `&str`, `&String`) resolves to the blanket + // `impl PartialEq<&B> for &A` (`core::cmp::impls`), whose body + // forwards to the pointee's own `eq`. Both operands already + // carry the single `SomeString` value the reference projects + // to, so this is the same `BinOp("eq")` / `BinOp("ne")` + // (`ll_streq`) the `::eq` arm above emits; + // `ne` is the same lowering: `AbstractStringRepr.rtype_ne` + // is `ll_streq` followed by `bool_not`, and the flow graph's + // string `BinOp("ne")` is that decomposition's canonical + // spelling. Left residual, + // every `w_str_get_wtf8(a) == w_str_get_wtf8(b)` + // and `name == "__doc__"` is an unlowered helper that blocks + // the descent into its caller (`is_w`, `getattr_str_impl`). + // The operand-type gate keeps the integer and float impls + // that share this owner (`::eq`) out. if args.len() == 2 - && fmt_path_ends_with(&segments, &["cmp", "impls", "", "ne"]) + && let [.., owner_a, owner_b, owner_c, leaf] = segments.as_slice() + && owner_a == "cmp" + && owner_b == "impls" + && owner_c == "" + && matches!(leaf.as_str(), "eq" | "ne") && [first_arg_ty.as_ref(), second_arg_ty.as_ref()] .iter() - .all(|ty| ty.is_some_and(|ty| tyref_strips_to_str(ty, self.llbc))) + .all(|t| t.is_some_and(|t| tyref_is_string_value(t, self.llbc))) { let res = self .graph @@ -10801,7 +10812,7 @@ impl<'a> Lowering<'a> { self.graph.block_mut(bb_id).operations.push(SpaceOperation { result: Some(res.clone()), kind: OpKind::BinOp { - op: "ne".to_string(), + op: leaf.clone(), lhs: args[0].clone(), rhs: args[1].clone(), result_ty: ValueType::Int, @@ -11739,7 +11750,8 @@ impl<'a> Lowering<'a> { // `Result`. RPython has a direct GCREF result plus // an implicit MemoryError edge, so retarget the validated // machine-count helper to that pointer ABI before `result_exc` erases - // the Result diamond. + // the Result diamond. The `_make_ovf2long` seams take the same shape + // with both operands already machine words. let op_kind = if let OpKind::Call { target: CallTarget::FunctionPath { segments }, args, @@ -11755,7 +11767,12 @@ impl<'a> Lowering<'a> { .as_ref() .is_some_and(|ty| tyref_to_value_type(ty, self.llbc) == ValueType::Int) } - Some("bigint_lshift_int_int_result") => { + Some( + "bigint_lshift_int_int_result" + | "bigint_add_int_int" + | "bigint_sub_int_int" + | "bigint_mul_int_int", + ) => { first_arg_ty .as_ref() .is_some_and(|ty| tyref_to_value_type(ty, self.llbc) == ValueType::Int) @@ -11766,6 +11783,7 @@ impl<'a> Lowering<'a> { _ => false, } && let Some(residual) = crate::front::rbigint_call::lshift_count_residual_path(segments) + .or_else(|| crate::front::rbigint_call::ovf2long_residual_path(segments)) { OpKind::Call { target: CallTarget::FunctionPath { segments: residual }, @@ -34894,6 +34912,40 @@ mod tests { ); } + /// Anchor the string-reference `PartialEq` fold to the real lowered IR of + /// `is_w` — `w_str_get_wtf8(w_one) == w_str_get_wtf8(w_two)` over two + /// `&Wtf8`. The comparison resolves to the blanket `&A == &B` impl in + /// `core::cmp::impls`, which must fold to `BinOp("eq")` like the direct + /// `::eq` spelling; left residual it is the unlowered + /// helper every descent through `is_w` (`len(x)` via `len_w`) stops at. + /// Ignored by default (loads the real LLBC). + #[test] + #[ignore] + fn string_ref_eq_fold_real_is_w() { + use crate::model::{CallTarget, OpKind}; + + let path = crate::runtime_names::artifacts::INTERPRETER_ULLBC; + let llbc = Llbc::load(path).expect("load real LLBC"); + let graph = super::lower_function(&llbc, "pyre_interpreter::baseobjspace::is_w") + .expect("lower is_w"); + let ops = || graph.blocks.iter().flat_map(|b| b.operations.iter()); + assert_eq!( + ops() + .filter(|op| matches!( + &op.kind, + OpKind::Call { target: CallTarget::FunctionPath { segments }, .. } + if super::fmt_path_ends_with(segments, &["cmp", "impls", "", "eq"]) + )) + .count(), + 0, + "no residual `core::cmp::impls::::eq` call survives the fold" + ); + assert!( + ops().any(|op| matches!(&op.kind, OpKind::BinOp { op, .. } if op == "eq")), + "the `&Wtf8` comparison lowers to a `BinOp(eq)`" + ); + } + /// `get_w_locals` is `getdebug_data().map_or(PY_NULL, |data| data.w_locals)` /// over `Option<&FrameDebugData>` — a SHARED-reference niche Option. Its /// `Some` payload must alias the base pointer (no aggregate `__pos_0` diff --git a/majit/majit-translate/src/front/rbigint_call.rs b/majit/majit-translate/src/front/rbigint_call.rs index 24d35d8a8a8..8d7e511a492 100644 --- a/majit/majit-translate/src/front/rbigint_call.rs +++ b/majit/majit-translate/src/front/rbigint_call.rs @@ -286,6 +286,40 @@ pub(crate) fn lshift_count_residual_path(segments: &[String]) -> Option Option> { + let residual = match segments.last().map(String::as_str) { + Some("bigint_add_int_int") => "jit_bigint_add_int_int", + Some("bigint_sub_int_int") => "jit_bigint_sub_int_int", + Some("bigint_mul_int_int") => "jit_bigint_mul_int_int", + _ => return None, + }; + if !segments + .iter() + .rev() + .skip(1) + .take(2) + .eq(["descroperation", "objspace"]) + { + return None; + } + Some( + [ + crate::runtime_names::crates::INTERPRETER, + "objspace", + "descroperation", + residual, + ] + .into_iter() + .map(str::to_string) + .collect(), + ) +} + /// RustPython's compiler exposes integer constants as an opaque foreign /// BigInt. Retarget the one sanctioned conversion seam to a pointer-returning /// pure residual so Malachite never enters the translated arithmetic graph. @@ -340,8 +374,8 @@ pub(crate) fn unop_wrapper_residual_path(segments: &[String]) -> Option Option> { let residual = match segments.last().map(String::as_str) { Some("bigint_floordiv_nonzero") => "jit_bigint_div_floor", @@ -693,4 +727,46 @@ mod tests { ); } } + + #[test] + fn maps_the_ovf2long_seams_and_nothing_else() { + for (source, residual) in [ + ("bigint_add_int_int", "jit_bigint_add_int_int"), + ("bigint_sub_int_int", "jit_bigint_sub_int_int"), + ("bigint_mul_int_int", "jit_bigint_mul_int_int"), + ] { + assert_eq!( + ovf2long_residual_path(&segs(&[ + crate::runtime_names::crates::INTERPRETER, + "objspace", + "descroperation", + source, + ])), + Some(segs(&[ + crate::runtime_names::crates::INTERPRETER, + "objspace", + "descroperation", + residual, + ])) + ); + } + // The owner is load-bearing, as it is for the unary wrappers. + assert_eq!( + ovf2long_residual_path(&segs(&[ + crate::runtime_names::crates::OBJECT, + "longobject", + "bigint_mul_int_int", + ])), + None, + ); + assert_eq!( + ovf2long_residual_path(&segs(&[ + crate::runtime_names::crates::INTERPRETER, + "objspace", + "descroperation", + "bigint_mul", + ])), + None, + ); + } } diff --git a/majit/majit-translate/src/front/result_exc.rs b/majit/majit-translate/src/front/result_exc.rs index c9ae790a7e7..050e4eb02a2 100644 --- a/majit/majit-translate/src/front/result_exc.rs +++ b/majit/majit-translate/src/front/result_exc.rs @@ -1021,6 +1021,7 @@ pub(crate) fn op_operand_vars(kind: &OpKind) -> Vec { | OpKind::AssertGreen { value, .. } | OpKind::IsConstant { value, .. } | OpKind::IsVirtual { value, .. } => vec![value.clone()], + OpKind::GuardClass { base } => vec![base.clone()], OpKind::VtableMethodPtr { receiver, .. } => vec![receiver.clone()], OpKind::IsInstance { obj, class_carrier, .. @@ -3553,10 +3554,15 @@ pub(crate) fn collapse_pos0_read( /// The one-argument `PyError` constructors this pass fuses, and the published /// helper each fuses into. /// -/// Measured over the 301 distinct gateway wrappers: `type_error` is the only -/// constructor that reaches a raise site, 681 occurrences, all of them in the -/// shape below. Extending the table is a one-line change plus its helper. -const FUSED_KIND_CTORS: &[(&str, &str)] = &[("type_error", "pyerror_type_error_to_exc_object")]; +/// Gateway wrappers contribute the `type_error` sites; the exact-int +/// `int_floordiv` / `int_mod` bodies contribute the literal-message +/// `zero_division` sites. Each entry removes the Rust carrier aggregate from +/// the generated JitCode while preserving the interpreter's exception-object +/// materialisation as one opaque call. +const FUSED_KIND_CTORS: &[(&str, &str)] = &[ + ("type_error", "pyerror_type_error_to_exc_object"), + ("zero_division", "pyerror_zero_division_to_exc_object"), +]; /// Fuse `PyError::(msg)` and the `pyerror_to_exc_object` that consumes /// it into a single published call. diff --git a/majit/majit-translate/src/generated.rs b/majit/majit-translate/src/generated.rs index 4587ec0bbb6..29ef11c6a7f 100644 --- a/majit/majit-translate/src/generated.rs +++ b/majit/majit-translate/src/generated.rs @@ -242,6 +242,7 @@ fn build() -> AllJitCodes { split_portal: false, }], register_trait_families: Vec::new(), + helper_graphs: Vec::new(), }, }, None, diff --git a/majit/majit-translate/src/inline.rs b/majit/majit-translate/src/inline.rs index 2ead446fe80..9dc2bd21fe0 100644 --- a/majit/majit-translate/src/inline.rs +++ b/majit/majit-translate/src/inline.rs @@ -766,6 +766,9 @@ pub(crate) fn remap_op_kind( value: remap_var(value), kind_char: *kind_char, }, + OpKind::GuardClass { base } => OpKind::GuardClass { + base: remap_var(base), + }, OpKind::IsInstance { obj, class_carrier, @@ -1047,6 +1050,7 @@ pub fn op_variable_refs(kind: &OpKind) -> Vec OpKind::AssertGreen { value, .. } | OpKind::IsConstant { value, .. } | OpKind::IsVirtual { value, .. } => vec![clone_var(value)], + OpKind::GuardClass { base } => vec![clone_var(base)], OpKind::IsInstance { obj, class_carrier, .. } => { @@ -1322,6 +1326,7 @@ pub fn is_pure_op(kind: &OpKind) -> bool { | OpKind::GuardTrue { .. } | OpKind::GuardFalse { .. } | OpKind::GuardValue { .. } + | OpKind::GuardClass { .. } | OpKind::AssertGreen { .. } | OpKind::IsConstant { .. } | OpKind::IsVirtual { .. } diff --git a/majit/majit-translate/src/lib.rs b/majit/majit-translate/src/lib.rs index 4fb3760132c..45c35f6c7eb 100644 --- a/majit/majit-translate/src/lib.rs +++ b/majit/majit-translate/src/lib.rs @@ -2149,6 +2149,9 @@ fn analyze_pipeline_from_module_paths( &config.pipeline.jit_drivers, &config.pipeline.transform.jitdriver_receiver_roots, ); + for path in &config.pipeline.helper_graphs { + call_control.register_helper_graph(path.clone()); + } // warmspot.py WarmRunnerDesc.make_virtualizable_infos — // assigns each registered driver's virtualizable metadata only after the // complete driver set exists. @@ -2915,6 +2918,7 @@ mod portal_driver_tests { transform: GraphTransformConfig::default(), jit_drivers: vec![driver(portal.clone())], register_trait_families: Vec::new(), + helper_graphs: Vec::new(), }; register_configured_jitdrivers( &mut call_control, @@ -2990,6 +2994,7 @@ mod portal_driver_tests { split_portal: true, }], register_trait_families: Vec::new(), + helper_graphs: Vec::new(), }; register_configured_jitdrivers( &mut call_control, diff --git a/majit/majit-translate/src/model.rs b/majit/majit-translate/src/model.rs index 014acc24c40..7ba8a47d6d0 100644 --- a/majit/majit-translate/src/model.rs +++ b/majit/majit-translate/src/model.rs @@ -1106,6 +1106,24 @@ pub enum OpKind { /// `codewriter/jtransform.py:611`. kind_char: char, }, + /// `jtransform.py handle_getfield_typeptr` — a read of the + /// object header's class word is not a load but a `guard_class`: the + /// tracer pins the receiver's class and the op's result is that class + /// as a constant, which is what lets every `isinstance`-shaped test + /// downstream fold instead of recording `getfield` + `ptr_eq` + + /// `guard_true`. The blackhole reads the word (`bhimpl_guard_class`, + /// `cpu.bh_classof`). + /// + /// pyre's class word is `PyObject.ob_type`, embedded first in every + /// object as `ob_header`; `heaptracker::is_header_word` knows it by + /// that name. The result register keeps the kind the read had — + /// `int` when the pointer was typed raw, `ref` when it was typed + /// GC — so the consumers of the original read are untouched; the + /// class-pointer hint ops (`record_exact_class/ri`) already carry the + /// class through the int bank. + GuardClass { + base: crate::flowspace::model::Variable, + }, /// Project a callee function pointer out of a `dyn Trait` receiver's /// vtable for the named method slot. Result is integer-typed so it /// can be fed to `int_guard_value` (RPython `jtransform.py`). diff --git a/majit/majit-translate/src/pipeline.rs b/majit/majit-translate/src/pipeline.rs index 6abdbc4a0cb..5563a89730c 100644 --- a/majit/majit-translate/src/pipeline.rs +++ b/majit/majit-translate/src/pipeline.rs @@ -97,6 +97,16 @@ pub struct PipelineConfig { /// disposition; a consumer opts its own traits in by name. #[serde(default)] pub register_trait_families: Vec, + /// Graphs seeded into the `find_all_graphs` BFS beside the portals. + /// + /// `call.py` seeds `support.inline_calls_to`: helper graphs the + /// portal never calls in source because the codewriter lowers an + /// operation straight to a residual call of that helper. A host that + /// lowers an opcode to a residual the same way names the residual's + /// body here, so the graph exists for a trace to descend + /// (`CallControl::register_helper_graph`). + #[serde(default)] + pub helper_graphs: Vec, } /// Result of running the full pipeline on a single function. @@ -313,6 +323,7 @@ mod tests { fn serialized_pipeline_config_requires_explicit_jit_drivers() { let config = PipelineConfig { transform: GraphTransformConfig::default(), + helper_graphs: Vec::new(), jit_drivers: vec![JitDriverSpec { portal: CallPath::from_segments(["engine", "mainloop"]), portal_runner: None, diff --git a/majit/majit-translate/src/translator/rtyper/call_registry.rs b/majit/majit-translate/src/translator/rtyper/call_registry.rs index ff42a24593b..c6a0a0044a7 100644 --- a/majit/majit-translate/src/translator/rtyper/call_registry.rs +++ b/majit/majit-translate/src/translator/rtyper/call_registry.rs @@ -116,7 +116,9 @@ pub(crate) fn is_exception_object_materializer(key: &FunctionPathKey) -> bool { }; if !matches!( leaf, - "pyerror_to_exc_object" | "pyerror_type_error_to_exc_object" + "pyerror_to_exc_object" + | "pyerror_type_error_to_exc_object" + | "pyerror_zero_division_to_exc_object" ) { return false; } @@ -1167,7 +1169,11 @@ mod tests { fn exception_materializers_publish_instance_result_signature() { let bk = Rc::new(Bookkeeper::new()); let registry = CallRegistry::new(bk); - for leaf in ["pyerror_to_exc_object", "pyerror_type_error_to_exc_object"] { + for leaf in [ + "pyerror_to_exc_object", + "pyerror_type_error_to_exc_object", + "pyerror_zero_division_to_exc_object", + ] { for path in [ vec!["pyre_interpreter", "error", leaf], vec!["error", leaf], diff --git a/majit/majit-translate/src/translator/rtyper/cutover.rs b/majit/majit-translate/src/translator/rtyper/cutover.rs index 681e3691415..f4463a6f5f7 100644 --- a/majit/majit-translate/src/translator/rtyper/cutover.rs +++ b/majit/majit-translate/src/translator/rtyper/cutover.rs @@ -6638,7 +6638,11 @@ mod tests { fn exception_materializer_stub_uses_semantic_instance_result() { use crate::annotator::model::SomeValue; - for name in ["pyerror_to_exc_object", "pyerror_type_error_to_exc_object"] { + for name in [ + "pyerror_to_exc_object", + "pyerror_type_error_to_exc_object", + "pyerror_zero_division_to_exc_object", + ] { let key = FunctionPathKey::from_segments(["pyre_interpreter", "error", name]); let shell = residual_stub_result_shell(&key, Some(OBJECTPTR_RETURN_TYPE)) .expect("exception materializer must have a result shell"); diff --git a/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs b/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs index e19baeb9222..984e9211e95 100644 --- a/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs +++ b/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs @@ -3284,6 +3284,7 @@ fn opkind_variant_name(kind: &OpKind) -> &'static str { OpKind::CurrentTraceLength => "CurrentTraceLength", OpKind::IsConstant { .. } => "IsConstant", OpKind::IsVirtual { .. } => "IsVirtual", + OpKind::GuardClass { .. } => "GuardClass", OpKind::IsInstance { .. } => "IsInstance", OpKind::ConditionalCall { .. } => "ConditionalCall", OpKind::ConditionalCallValue { .. } => "ConditionalCallValue", @@ -3329,6 +3330,7 @@ fn post_rtyper_jtransform_variant_name(kind: &OpKind) -> Option<&'static str> { OpKind::CurrentTraceLength => "CurrentTraceLength (jtransform.py:1731-1743)", OpKind::IsConstant { .. } => "IsConstant (jtransform.py:1731-1743)", OpKind::IsVirtual { .. } => "IsVirtual (jtransform.py:1731-1743)", + OpKind::GuardClass { .. } => "GuardClass (jtransform.py:1004-1010)", OpKind::ConditionalCall { .. } => "ConditionalCall (jtransform.py:1665-1688)", OpKind::ConditionalCallValue { .. } => "ConditionalCallValue (jtransform.py:1665-1688)", OpKind::RecordKnownResult { .. } => "RecordKnownResult (jtransform.py:292-313)", diff --git a/majit/majit-translate/src/translator/rtyper/legacy_annotator.rs b/majit/majit-translate/src/translator/rtyper/legacy_annotator.rs index 44ad231bf54..01c91bce8b6 100644 --- a/majit/majit-translate/src/translator/rtyper/legacy_annotator.rs +++ b/majit/majit-translate/src/translator/rtyper/legacy_annotator.rs @@ -402,6 +402,8 @@ fn infer_op_type(kind: &OpKind) -> ValueType { OpKind::IsInstance { .. } => ValueType::Bool, // RPython: vtable entry is a `Ptr(FuncType)` address. OpKind::VtableMethodPtr { .. } => ValueType::Int, + // The class pointer, as the hint ops carry it. + OpKind::GuardClass { .. } => ValueType::Int, OpKind::IndirectCall { result_ty, .. } => result_ty.clone(), OpKind::CallElidable { result_kind, .. } | OpKind::CallResidual { result_kind, .. } diff --git a/majit/majit-translate/tests/test_rbigint_mir.rs b/majit/majit-translate/tests/test_rbigint_mir.rs index e8908ac6509..e1af689fb06 100644 --- a/majit/majit-translate/tests/test_rbigint_mir.rs +++ b/majit/majit-translate/tests/test_rbigint_mir.rs @@ -1295,7 +1295,7 @@ const UNARY_RESIDUAL_CALLERS: &[(&str, &str)] = &[ /// Graphs the test reaches for one at a time rather than through a table. const SINGLE_GRAPHS: &[&str] = &[ - "bigint_add", + "bigint_and", "long_int_compare", "long_pow", "long_lshift", @@ -1401,10 +1401,10 @@ fn dependent_crate_rbigint_identity_retargets_opaque_llbc_declaration() { .functions .iter() .find(|function| { - function.name == "bigint_add" + function.name == "bigint_and" && function.module_path.ends_with("objspace::descroperation") }) - .expect("descroperation::bigint_add graph"); + .expect("descroperation::bigint_and graph"); let calls: Vec> = helper .graph @@ -1444,7 +1444,7 @@ fn dependent_crate_rbigint_identity_retargets_opaque_llbc_declaration() { "pyre_interpreter", "objspace", "descroperation", - "jit_bigint_add", + "jit_bigint_and", ] }) .count(); diff --git a/majit/majit-translate/tests/test_result_exc_lowering.rs b/majit/majit-translate/tests/test_result_exc_lowering.rs index 6301273182d..0ee998c1439 100644 --- a/majit/majit-translate/tests/test_result_exc_lowering.rs +++ b/majit/majit-translate/tests/test_result_exc_lowering.rs @@ -501,7 +501,9 @@ fn raise_path_calls(name: &str) -> (usize, usize, usize) { continue; }; match segments.last().map(String::as_str) { - Some("pyerror_type_error_to_exc_object") => fused += 1, + Some( + "pyerror_type_error_to_exc_object" | "pyerror_zero_division_to_exc_object", + ) => fused += 1, Some("pyerror_to_exc_object") => materialise += 1, Some(_) if segments.len() >= 2 && segments[segments.len() - 2] == "PyError" => { ctors += 1 @@ -564,6 +566,23 @@ fn gateway_wrapper_refusals_all_residualize() { } } +#[test] +fn exact_int_zero_division_raise_sites_fuse_their_constructor() { + // `int_floordiv` and `int_mod` are the two exact-int operator bodies that + // raise ZeroDivisionError. Their shared literal message must reach the + // fused materialiser so the generated descent carries a + // `W_BaseException`, never an in-trace Rust `PyError` aggregate. + for name in ["int_floordiv", "int_mod"] { + let (fused, materialise, ctors) = raise_path_calls(name); + assert!(fused > 0, "{name}: zero-division fusion must fire"); + assert_eq!(ctors, 0, "{name}: no PyError constructor may survive"); + assert_eq!( + materialise, 0, + "{name}: no unfused materialisation may survive" + ); + } +} + #[test] fn formatted_message_raise_sites_keep_the_two_call_form() { // `__class_getitem__`'s checks build their message with `format!`, whose diff --git a/pyre/bench/nbody.wasm.jitstats b/pyre/bench/nbody.wasm.jitstats index 16daf2b7fbe..0f58b3de18f 100644 --- a/pyre/bench/nbody.wasm.jitstats +++ b/pyre/bench/nbody.wasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1608 +guard_failures=1607 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 diff --git a/pyre/bench/synth/arith_int_bool.cranelift.jitstats b/pyre/bench/synth/arith_int_bool.cranelift.jitstats index 04176f6ee47..a04ffde2640 100644 --- a/pyre/bench/synth/arith_int_bool.cranelift.jitstats +++ b/pyre/bench/synth/arith_int_bool.cranelift.jitstats @@ -6,6 +6,7 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 diff --git a/pyre/bench/synth/arith_int_bool.dynasm.jitstats b/pyre/bench/synth/arith_int_bool.dynasm.jitstats index 04176f6ee47..a04ffde2640 100644 --- a/pyre/bench/synth/arith_int_bool.dynasm.jitstats +++ b/pyre/bench/synth/arith_int_bool.dynasm.jitstats @@ -6,6 +6,7 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 diff --git a/pyre/bench/synth/arith_int_bool.py b/pyre/bench/synth/arith_int_bool.py index 8123ab90841..232383f092b 100644 --- a/pyre/bench/synth/arith_int_bool.py +++ b/pyre/bench/synth/arith_int_bool.py @@ -1,5 +1,6 @@ # pyre-check: max-pypy-ratio=381 # pyre-check: jitstats-band=guard_failures=2 +# pyre-check: spec-folds=binary_op_descent,compare_op_descent # One host disagrees with the other two by a single count: at the same # commit the ubuntu-24.04 and windows-latest legs read dynasm # guard_failures=2507 and macos-latest read 2508, while loops_compiled=7 and diff --git a/pyre/bench/synth/arith_int_bool.wasm.jitstats b/pyre/bench/synth/arith_int_bool.wasm.jitstats index a65cb30e67e..1f3636376ce 100644 --- a/pyre/bench/synth/arith_int_bool.wasm.jitstats +++ b/pyre/bench/synth/arith_int_bool.wasm.jitstats @@ -6,6 +6,7 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 diff --git a/pyre/bench/synth/builtin_len_descent.cranelift.jitstats b/pyre/bench/synth/builtin_len_descent.cranelift.jitstats new file mode 100644 index 00000000000..8ea33c65cd2 --- /dev/null +++ b/pyre/bench/synth/builtin_len_descent.cranelift.jitstats @@ -0,0 +1,19 @@ +bridges_compiled=13 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=2615 +internal_compile_panics=0 +loops_aborted=0 +loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/builtin_len_descent.dynasm.jitstats b/pyre/bench/synth/builtin_len_descent.dynasm.jitstats new file mode 100644 index 00000000000..8ea33c65cd2 --- /dev/null +++ b/pyre/bench/synth/builtin_len_descent.dynasm.jitstats @@ -0,0 +1,19 @@ +bridges_compiled=13 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=2615 +internal_compile_panics=0 +loops_aborted=0 +loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/builtin_len_descent.py b/pyre/bench/synth/builtin_len_descent.py new file mode 100644 index 00000000000..e708e481abc --- /dev/null +++ b/pyre/bench/synth/builtin_len_descent.py @@ -0,0 +1,54 @@ +# pyre-check: spec-folds=builtin_len_descent +# PyPy installs the builtin implementations reached by +# descroperation.py `_len` through typedef.py +# `use_special_method_shortcut('__len__')`. Keep every layout formerly emitted +# by the hand-written builtin_len fold on the generated gateway descent: list +# strategies, ordinary and specialised pairs, set/frozenset, range, unicode, +# bytes, and mutable bytearray. The shared inner loop deliberately retraces +# across layouts; every result contributes to one checksum. + + +def pair(a, b): + return (a, b) + + +def hot(n, objects): + total = 0 + for obj in objects: + for _ in range(n): + total += len(obj) + return total + + +def main(): + try: + import pypyjit + + pypyjit.set_param("threshold=20,function_threshold=20") + except ImportError: + pass + + marker = object() + objects = ( + [1, 2, 3, 4], + [1.25, 2.5], + # Mixed numbers take the IntOrFloat strategy, whose length read is the + # Integer strategy's own. + [1, 2.5], + [marker], + [], + (1, 2, 3), + pair(1, 2), + pair(1.0, 2.0), + pair(marker, marker), + {1, 2, 3}, + frozenset((1, 2)), + range(11), + "abc", + b"ab", + bytearray(b"abcd"), + ) + print(hot(4000, objects)) + + +main() diff --git a/pyre/bench/synth/builtin_len_descent.wasm.jitstats b/pyre/bench/synth/builtin_len_descent.wasm.jitstats new file mode 100644 index 00000000000..8ea33c65cd2 --- /dev/null +++ b/pyre/bench/synth/builtin_len_descent.wasm.jitstats @@ -0,0 +1,19 @@ +bridges_compiled=13 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=2615 +internal_compile_panics=0 +loops_aborted=0 +loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/dict_set.wasm.jitstats b/pyre/bench/synth/dict_set.wasm.jitstats index c37f2cc744f..da31ee4c207 100644 --- a/pyre/bench/synth/dict_set.wasm.jitstats +++ b/pyre/bench/synth/dict_set.wasm.jitstats @@ -12,7 +12,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=608 +guard_failures=607 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 diff --git a/pyre/bench/synth/fast_local_swap.py b/pyre/bench/synth/fast_local_swap.py index 8fc069b4d4b..c2523e92132 100644 --- a/pyre/bench/synth/fast_local_swap.py +++ b/pyre/bench/synth/fast_local_swap.py @@ -1,11 +1,12 @@ -# pyre-check: max-pypy-ratio=2 +# pyre-check: max-pypy-ratio=2.6 # The ceiling gates cranelift as well as dynasm, and `perf_gate_floor` derives # a floor from it as ceiling/6, so both ends of the reading spread pick it. Run # 33384229844 reads 0.5x (macos dynasm, median of 3), 0.6x (macos cranelift), # 1.3x and 1.5x (ubuntu) on the four pairs where pypy's baseline was measurable -# -- wasm is ungated and windows read a clamped baseline. Sizing off the slow -# end alone lands the floor on the fast end: 3x derives exactly 0.5x. 2x clears -# both by a third. +# -- wasm is ungated and windows read a clamped baseline. Run 33860926996 later +# read 2.2x on ubuntu with pyre at 0.33s, the same as main's 0.34s in run +# 33849050216; pypy's denominator moved instead. 2.6 is that cross-host high +# plus 15%, while its 0.433x derived floor remains below the measured 0.5x. # pyre-check: skip-cpython # cpython 1.33s vs pyre 0.24s (5.5x on the ubuntu runner), and it is not # gated on — only pypy is. diff --git a/pyre/bench/synth/force_all_frames_hot_stack.py b/pyre/bench/synth/force_all_frames_hot_stack.py index f4989ca8074..3a1993f0682 100644 --- a/pyre/bench/synth/force_all_frames_hot_stack.py +++ b/pyre/bench/synth/force_all_frames_hot_stack.py @@ -1,5 +1,5 @@ # pyre-check: selfcheck -# pyre-check: spec-folds=for_iter_next,compare_op_int +# pyre-check: spec-folds=for_iter_next,compare_op_descent # pyre-check: selfcheck-compiles=hot # Self-checking regression guard for `force_all_frames` # (`executioncontext.rs`), the only frame-materializing consumer in the tree diff --git a/pyre/bench/synth/foriter_bridge_walk_keeps_the_iteration.py b/pyre/bench/synth/foriter_bridge_walk_keeps_the_iteration.py index 867f5b04607..326069ef86e 100644 --- a/pyre/bench/synth/foriter_bridge_walk_keeps_the_iteration.py +++ b/pyre/bench/synth/foriter_bridge_walk_keeps_the_iteration.py @@ -3,8 +3,10 @@ # Three spellings of one loop over four values. They differ only in the # expression the `for` iterates, so the three totals must agree. # -# `trace_limit` is sized so one traced iteration overflows it and the walk -# aborts with the iterator already advanced. A bridge/retrace recording that +# `trace_limit` is sized so the walk overflows it before it can close the +# loop (one iteration records ~225 ops here, so the first attempt runs into a +# second pass) and aborts with the iterator already advanced; the retry then +# crosses the merge point past 0.8x the limit and segments. A bridge/retrace recording that # does not commit restores the cursor it advanced eagerly, but only the range # and zip FOR_ITER specializations journalled theirs: the list and tuple # iterators reach the generic `for_iter_next` residual, so nothing was recorded @@ -17,7 +19,10 @@ try: import pypyjit - pypyjit.set_param("trace_limit=300") + # The 0.8x window is a raw-op-count property; every backend records the + # same raw op count for this body, and the sweep put the window around + # 220-260. + pypyjit.set_param("trace_limit=240") pypyjit.set_param("threshold=20") except ImportError: pass diff --git a/pyre/bench/synth/foriter_root_walk_keeps_the_iteration.py b/pyre/bench/synth/foriter_root_walk_keeps_the_iteration.py index a2ca7c5605f..ab235581b9f 100644 --- a/pyre/bench/synth/foriter_root_walk_keeps_the_iteration.py +++ b/pyre/bench/synth/foriter_root_walk_keeps_the_iteration.py @@ -17,10 +17,15 @@ try: import pypyjit - # The direct list-iterator fold closes at 306 operations. PyPy keeps the - # same abort/segment counts at this limit as at the former 300-operation - # boundary, while still compiling the exact-list loop. - pypyjit.set_param("trace_limit=306") + # Sized to this branch's raw op count: one traced iteration records fewer + # ops here than on main (where the direct list-iterator fold closes at 306 + # and the limit sits there), so the 0.8x segmenting cut needs a lower + # limit to be crossed at all. See foriter_bridge_walk_keeps_the_iteration. + + # The 0.8x window is a raw-op-count property; every backend records the + # same raw op count for this body, and the list and tuple arms' windows + # overlap around 240. + pypyjit.set_param("trace_limit=240") pypyjit.set_param("threshold=20") except ImportError: pass diff --git a/pyre/bench/synth/foriter_segment_cut_resumes_forward.py b/pyre/bench/synth/foriter_segment_cut_resumes_forward.py index ac453fbf5c1..5cfc8c1f479 100644 --- a/pyre/bench/synth/foriter_segment_cut_resumes_forward.py +++ b/pyre/bench/synth/foriter_segment_cut_resumes_forward.py @@ -22,14 +22,10 @@ try: import pypyjit - # Fit to one traced iteration's cost the same way - # `trace_segmenting_over_limit_retry` is: it read 330 while the FOR_ITER - # receiver pin was spelled `ptr_eq` + `guard_true`, and `guard_value` - # shortened the body out from under it. The census this file gates on - # (`loop loop_genexp`, `loop prefix_next`) is the same at 300, 315 and 330; - # 315 is where the cut shape it exercises comes back - # (`bridges_compiled=26 loops_aborted=29` against 19/40 at 330). - pypyjit.set_param("trace_limit=315") + # The 0.8x window is a raw-op-count property; every backend records the + # same raw op count for this body, and the sweep put the window around + # 220-260. + pypyjit.set_param("trace_limit=240") pypyjit.set_param("threshold=20") except ImportError: pass diff --git a/pyre/bench/synth/generator_tree_recursion.py b/pyre/bench/synth/generator_tree_recursion.py index 1957001d9e3..c0c6df1be73 100644 --- a/pyre/bench/synth/generator_tree_recursion.py +++ b/pyre/bench/synth/generator_tree_recursion.py @@ -1,19 +1,22 @@ -# pyre-check: max-pypy-ratio=14 +# pyre-check: max-pypy-ratio=23 # pyre-check: jitstats-band=guard_failures=8 # Successful bridge closure and a pre-trace Decline are not aborts in # `MetaInterp._interpret`. Charging both to pyre's local abort ceiling held this # fixture at 29 bridges / 3600 guard failures; the corrected lifecycle reaches -# 48 / 7378. The PyPy oracle compiles still more (65 bridges) with forcings=0, +# 33 / 4382. The PyPy oracle compiles still more (65 bridges) with forcings=0, # virtualizables forced=0 and nvirtuals=721, so the higher count is coverage, -# not a regression to suppress. Four final-binary cranelift runs measured -# 12.5x..12.7x; 14x leaves 10% headroom. The recovery target is PyPy's -# zero-forcing per-`MIFrame` recursive-frame/blackhole path, not restoring the -# abort-ceiling shortcut. +# not a regression to suppress. Once the manual arithmetic folds were retired +# onto generated interpreter descents, branch runs 33692288311, 33813140363 and +# 33860926996 measured cranelift at 13.6x..19.7x and dynasm at 10.2x..13.8x +# while the exact 3-loop / 33-bridge / 4382-guard shape stayed fixed. 23x is the +# cross-host high plus 15%. The recovery target is PyPy's canonical codewriter +# inline call to the arithmetic body and its zero-forcing per-`MIFrame` +# recursive-frame/blackhole path, not either retired shortcut. # Jitcounter decay is 0.96 every 32 minor collections # (majit-trace/src/counter.rs), so guard_failures tracks collection count during # each guard's warm-up rather than a compile decision. One host measured # 3648..3661 across nursery sizes before the lifecycle fix; decay=0 now pins -# 7378 everywhere, while loops_compiled=3 and bridges_compiled=48 remain +# 4382 everywhere, while loops_compiled=3 and bridges_compiled=33 remain # gated exactly. The fixture sets decay=0 itself, so the band covers the pinned # run, not that 13-wide unpinned spread; width 8 is margin (0.22%). Real # regressions this gate caught moved by hundreds to thousands (828 -> 4923, diff --git a/pyre/bench/synth/instance_surrogate_attrs.cranelift.jitstats b/pyre/bench/synth/instance_surrogate_attrs.cranelift.jitstats index 6bd97d5454e..e5d4b97cc22 100644 --- a/pyre/bench/synth/instance_surrogate_attrs.cranelift.jitstats +++ b/pyre/bench/synth/instance_surrogate_attrs.cranelift.jitstats @@ -15,5 +15,5 @@ field_pos_spec_misplaced=0 guard_failures=1 internal_compile_panics=0 loops_aborted=0 -loops_compiled=2 +loops_compiled=1 retraces_compiled=0 diff --git a/pyre/bench/synth/instance_surrogate_attrs.dynasm.jitstats b/pyre/bench/synth/instance_surrogate_attrs.dynasm.jitstats index 6bd97d5454e..e5d4b97cc22 100644 --- a/pyre/bench/synth/instance_surrogate_attrs.dynasm.jitstats +++ b/pyre/bench/synth/instance_surrogate_attrs.dynasm.jitstats @@ -15,5 +15,5 @@ field_pos_spec_misplaced=0 guard_failures=1 internal_compile_panics=0 loops_aborted=0 -loops_compiled=2 +loops_compiled=1 retraces_compiled=0 diff --git a/pyre/bench/synth/instance_surrogate_attrs.py b/pyre/bench/synth/instance_surrogate_attrs.py index e01ceaf0ac4..978d7b3c044 100644 --- a/pyre/bench/synth/instance_surrogate_attrs.py +++ b/pyre/bench/synth/instance_surrogate_attrs.py @@ -1,4 +1,9 @@ # pyre-check: max-pypy-ratio=50 +# pyre-check: spec-folds=builtin_getattr +# Keep the WTF-8 name through the same descriptor protocol as ordinary +# getattr: the bound Method uses the existing allocation/virtualization path, +# and the property enters its Python getter through the existing sub-walk. +# Neither access should remain an opaque call in the steady loop. N = 200000 METH = '\udc81' # lone surrogate naming a method on the class diff --git a/pyre/bench/synth/instance_surrogate_attrs.wasm.jitstats b/pyre/bench/synth/instance_surrogate_attrs.wasm.jitstats index 6bd97d5454e..e5d4b97cc22 100644 --- a/pyre/bench/synth/instance_surrogate_attrs.wasm.jitstats +++ b/pyre/bench/synth/instance_surrogate_attrs.wasm.jitstats @@ -15,5 +15,5 @@ field_pos_spec_misplaced=0 guard_failures=1 internal_compile_panics=0 loops_aborted=0 -loops_compiled=2 +loops_compiled=1 retraces_compiled=0 diff --git a/pyre/bench/synth/load_name_builtin_cell_fold.py b/pyre/bench/synth/load_name_builtin_cell_fold.py index 4ae7fd566ea..d6da2eec903 100644 --- a/pyre/bench/synth/load_name_builtin_cell_fold.py +++ b/pyre/bench/synth/load_name_builtin_cell_fold.py @@ -1,11 +1,11 @@ # pyre-check: max-pypy-ratio=2.5 -# pyre-check: spec-folds=builtin_len +# pyre-check: spec-folds=builtin_len_descent # pyre-check: skip-cpython # A module-scope LOAD_NAME whose name misses the module dict resolves through # the frame's builtin module. The builtins cell folds under the module dict's # version? (so a later global binding shadows the builtin) and the builtins # dict's own version?. The second loop proves that invalidation is seen: the -# census reads `builtin_len` consulted twice and fired once, the decline being +# census reads `builtin_len_descent` consulted twice and fired once, the decline being # the shadowed call. N = 225000000 diff --git a/pyre/bench/synth/locals_expansion_trace_too_long.py b/pyre/bench/synth/locals_expansion_trace_too_long.py index 1e9423dc40d..fd69814d07e 100644 --- a/pyre/bench/synth/locals_expansion_trace_too_long.py +++ b/pyre/bench/synth/locals_expansion_trace_too_long.py @@ -17,14 +17,15 @@ # then produces is the one `locals_in_wide_portal_frame` records for the same # frame. # -# The limit is 200 and the frame is 44 slots wide, so one expansion is worth -# roughly 88 ops -- enough to cross it partway through. Measured on release -# dynasm: +# The limit is 180 and the frame is 44 slots wide, so one expansion is worth +# roughly 88 ops -- enough to cross it partway through. The limit tracks what +# the walk records ahead of the expansion, so it is re-fit by sweeping whenever +# that moves, never nudged. Measured on release dynasm: # # builtin_locals ..._trace_limit_cut abrt_too_long # with the cut consulted=10 consulted=5 fired=5 5 # fired=5 -# cut suppressed consulted=10 suppressed=80 5 +# cut suppressed consulted=10 suppressed=45 5 # fired=10 # # Five of the ten expansions now end inside the unroll instead of emitting all @@ -59,7 +60,7 @@ # thing under test to one side of it. import pypyjit -pypyjit.set_param("trace_limit=200") +pypyjit.set_param("trace_limit=180") import sys diff --git a/pyre/bench/synth/mapdict_unboxed_float_write_hot.cranelift.jitstats b/pyre/bench/synth/mapdict_unboxed_float_write_hot.cranelift.jitstats index 6a532386981..1f05e121d55 100644 --- a/pyre/bench/synth/mapdict_unboxed_float_write_hot.cranelift.jitstats +++ b/pyre/bench/synth/mapdict_unboxed_float_write_hot.cranelift.jitstats @@ -6,6 +6,7 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 diff --git a/pyre/bench/synth/mapdict_unboxed_float_write_hot.dynasm.jitstats b/pyre/bench/synth/mapdict_unboxed_float_write_hot.dynasm.jitstats index 6a532386981..1f05e121d55 100644 --- a/pyre/bench/synth/mapdict_unboxed_float_write_hot.dynasm.jitstats +++ b/pyre/bench/synth/mapdict_unboxed_float_write_hot.dynasm.jitstats @@ -6,6 +6,7 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 diff --git a/pyre/bench/synth/mapdict_unboxed_float_write_hot.wasm.jitstats b/pyre/bench/synth/mapdict_unboxed_float_write_hot.wasm.jitstats index 6a532386981..1f05e121d55 100644 --- a/pyre/bench/synth/mapdict_unboxed_float_write_hot.wasm.jitstats +++ b/pyre/bench/synth/mapdict_unboxed_float_write_hot.wasm.jitstats @@ -6,6 +6,7 @@ fbw_blackhole_adopted_multi_frame=0 fbw_blackhole_adopted_single_frame=0 fbw_escape_plain_fallback=0 fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 fbw_midbody_latch_new_unjournaled=0 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 diff --git a/pyre/bench/synth/recursion_memo_branch.py b/pyre/bench/synth/recursion_memo_branch.py index 97fd3905afd..2f17afa7043 100644 --- a/pyre/bench/synth/recursion_memo_branch.py +++ b/pyre/bench/synth/recursion_memo_branch.py @@ -1,7 +1,7 @@ -# pyre-check: spec-folds=binary_op_int,truth_bool,compare_op_int +# pyre-check: spec-folds=binary_op_descent,truth_bool,compare_op_descent # The three folds that carry most of the corpus's fold traffic, and nothing # declared any of them, so switching one off was a silent change. This fixture -# fires them 398/69/65 times, the widest margin of any fixture firing all three. +# fires all three repeatedly, the widest margin of any fixture doing so. # Memoized vs plain recursion with post-warm-up branch divergence. The # memo-dict store (memo[n] = r) once died with a TypeError after warm-up # (an empty-string type name from a clobbered class read on the dict-store diff --git a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.py b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.py index af1531a9199..31e3c2a8a9c 100644 --- a/pyre/bench/synth/recursion_past_unroll_bound_from_loop.py +++ b/pyre/bench/synth/recursion_past_unroll_bound_from_loop.py @@ -1,4 +1,8 @@ -# pyre-check: max-pypy-ratio=6 +# pyre-check: max-pypy-ratio=9.1 +# Run 33860926996 measured the generated-descent cranelift path at 7.9x on a +# usable pypy baseline; 9.1x is that reading plus 15%. The recovery is the +# canonical codewriter inline call from `ArithmeticOpcodeHandler::binary_value` +# to the selected object-space body, not restoring the retired binary-op fold. # A recursion deeper than the inline unroll bound, driven from a loop body. # # `step` recurses nine frames deep, two past `FBW_MAX_INLINE_RECURSION`, so the diff --git a/pyre/bench/synth/str_getitem_len_hot.py b/pyre/bench/synth/str_getitem_len_hot.py index 0cde812a7c8..1c039090830 100644 --- a/pyre/bench/synth/str_getitem_len_hot.py +++ b/pyre/bench/synth/str_getitem_len_hot.py @@ -1,4 +1,4 @@ -# pyre-check: spec-folds=builtin_len +# pyre-check: spec-folds=builtin_len_descent # Hot-loop str/unicode subscript and length over every string kind: ASCII / # latin1 (1-byte code units), BMP (2-byte), and non-BMP astral (4-byte). The # subscripts emit residual STRGETITEM/UNICODEGETITEM and the lengths STRLEN/ @@ -12,8 +12,8 @@ # `bytesobject.py` takes off `_value` — and the `bytearray` arm reads # `W_BytearrayObject.length`, which is what `bytearrayobject.py`'s `_len` # reaches through `self._data` as `rlist.py`'s `("length", Signed)`. Without -# those arms the call stays an opaque forcing residual; `spec-folds` catches -# that regression directly. +# those paths the call stays an opaque forcing residual; `spec-folds` catches +# a regression in the generated builtin descent directly. # # `hot_mutating_len` is a correctness leg, not a speed one. `bytearray`'s # length is MUTABLE, so the compiled loop re-reads the field instead of @@ -71,8 +71,8 @@ def main(): print("bmp", hot_checksum(n, bmp_s)) print("astral", hot_checksum(n, astral_s)) print("len", hot_len(n, ascii_s), hot_len(n, bmp_s), hot_len(n, astral_s)) - # The fold census verifies the immutable bytes and mutable bytearray length - # arms directly; each leg only needs to stay hot enough to compile. + # The descent census verifies the immutable bytes and mutable bytearray + # length bodies directly; each leg only needs to stay hot enough to compile. bn = 5000 print("blen", hot_len(bn, ascii_b), hot_len(bn, short_b)) ban = 5000 diff --git a/pyre/bench/synth/trace_segmenting_over_limit_retry.py b/pyre/bench/synth/trace_segmenting_over_limit_retry.py index 3e40f1245ca..74812251fe6 100644 --- a/pyre/bench/synth/trace_segmenting_over_limit_retry.py +++ b/pyre/bench/synth/trace_segmenting_over_limit_retry.py @@ -17,18 +17,13 @@ # The 0.8x check only has something to fire at if a merge point is CROSSED while # the trace sits in the 0.8x..1.0x band, so the body size and `trace_limit` are # load-bearing together, not independently: at `trace_limit=200` the same body -# jumps the band in one crossing and never segments. +# jumps the band in one crossing and never segments. The band follows the RAW +# op count of one traced iteration, so a change in what the walker records for +# the body moves it. The swept band is recorded once, at the `set_param` +# call below, and 250 sits inside it. # # Which means the number tracks what one traced iteration COSTS in this tree, -# and has to be re-fit whenever that moves. Swept against the recorded shape -# (`loops_compiled=2 bridges_compiled=4 loops_aborted=9 guard_failures=1008`), -# the window is 270..288 and 280 is its centre; 264 misses the band low and -# never segments (`loops_aborted=64 guard_failures=11922`), 294 crosses it high -# and settles on a worse three-loop shape. It read 300 while the FOR_ITER -# receiver pin was spelled `ptr_eq` + `guard_true`; `guard_value` lets the -# optimizer constant-fold the class word out of the body, the iteration got -# shorter, and the whole window moved down with it. Re-fit it by sweeping, -# not by nudging. +# and has to be re-fit whenever that moves — by sweeping, not by nudging. # # CPython (the oracle) has no `pypyjit`; PyPy and pyre do. Guarding the import # keeps the output identical across all three while the params only bind where a @@ -37,7 +32,10 @@ try: import pypyjit - pypyjit.set_param("trace_limit=280") + # The 0.8x..1.0x band is a raw-op-count property; every backend records + # the same raw op count for this body, and the sweep put the band at + # 220..260. + pypyjit.set_param("trace_limit=250") pypyjit.set_param("threshold=20") except ImportError: pass diff --git a/pyre/bench/synth/unary_int_loop_carried.py b/pyre/bench/synth/unary_int_loop_carried.py index 21ee2df0cc1..190bff10d57 100644 --- a/pyre/bench/synth/unary_int_loop_carried.py +++ b/pyre/bench/synth/unary_int_loop_carried.py @@ -1,12 +1,11 @@ # pyre-check: max-pypy-ratio=6 -# The ceiling sits between the two measured states: folded this runs 2.5x -# pypy, and with `unary_invert_int` suppressed about 15.8x. +# Negative and invert are emitted as canonical codewriter `inline_call`s and +# carry no residual-call fold gate. # Unary operations must observe the current loop-carried integer. Exercise # both ordinary values and the large-integer boundary, plus neighboring # operations that serve as controls. Deterministic. -# A hot `~i` loop is folded here too: without the `unary_invert_int` fold each -# iteration leaves a `CallMayForce` residual instead of an `IntInvert`, which -# measures 6.9x on its own (0.095s -> 0.653s). +# A hot `~i` loop proves that generation now reaches the interpreter body's +# `IntInvert` directly instead of leaving a `CallMayForce` residual. def loop_carried_neg(n): @@ -44,7 +43,7 @@ def controls(n): def hot_invert(n): - """Hot `~int`, the `unary_invert_int` fold.""" + """Hot `~int`, served by the interpreter-body invert descent.""" s = 0 i = 0 while i < n: diff --git a/pyre/bench/synth/unary_long_descent.cranelift.jitstats b/pyre/bench/synth/unary_long_descent.cranelift.jitstats new file mode 100644 index 00000000000..910bd214d78 --- /dev/null +++ b/pyre/bench/synth/unary_long_descent.cranelift.jitstats @@ -0,0 +1,19 @@ +bridges_compiled=0 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=2 +internal_compile_panics=0 +loops_aborted=0 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/unary_long_descent.dynasm.jitstats b/pyre/bench/synth/unary_long_descent.dynasm.jitstats new file mode 100644 index 00000000000..910bd214d78 --- /dev/null +++ b/pyre/bench/synth/unary_long_descent.dynasm.jitstats @@ -0,0 +1,19 @@ +bridges_compiled=0 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=2 +internal_compile_panics=0 +loops_aborted=0 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/unary_long_descent.py b/pyre/bench/synth/unary_long_descent.py new file mode 100644 index 00000000000..39b16fa1c62 --- /dev/null +++ b/pyre/bench/synth/unary_long_descent.py @@ -0,0 +1,28 @@ +# Keep both operands loop-carried and outside the machine-int range. The fold +# layer never served these arms: canonical codewriter inline-calls prove that +# the whole `pos` and `invert` interpreter bodies admit exact +# W_LongObject operands instead of merely preserving former exact-int paths. +N = 200000 +LONG = 1 << 70 + + +def positive_long(n): + x = LONG + i = 0 + while i < n: + x = +x + i += 1 + return x + + +def invert_long(n): + x = LONG + i = 0 + while i < n: + x = ~x + i += 1 + return x + + +print(positive_long(N)) +print(invert_long(N)) diff --git a/pyre/bench/synth/unary_long_descent.wasm.jitstats b/pyre/bench/synth/unary_long_descent.wasm.jitstats new file mode 100644 index 00000000000..910bd214d78 --- /dev/null +++ b/pyre/bench/synth/unary_long_descent.wasm.jitstats @@ -0,0 +1,19 @@ +bridges_compiled=0 +descr_set_absent=0 +descr_set_ambiguous=0 +descr_set_stale_absent=0 +fbw_blackhole_adopted_multi_frame=0 +fbw_blackhole_adopted_single_frame=0 +fbw_escape_plain_fallback=0 +fbw_escape_plain_fallback_unclean=0 +fbw_foriter_item_dropped=0 +fbw_midbody_latch_new_unjournaled=0 +fbw_rolled_back_with_effects=0 +fbw_store_journal_rollback_failed=0 +field_pos_attached_misplaced=0 +field_pos_spec_misplaced=0 +guard_failures=2 +internal_compile_panics=0 +loops_aborted=0 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/unary_negative.py b/pyre/bench/synth/unary_negative.py index 459b908483f..ccceff1e108 100644 --- a/pyre/bench/synth/unary_negative.py +++ b/pyre/bench/synth/unary_negative.py @@ -1,7 +1,6 @@ # pyre-check: max-pypy-ratio=4.8 # Ubuntu run 33279264115: 2-2.4x; the ceiling is twice the slowest, # rounded up to one decimal place. -# pyre-check: spec-folds=unary_negative_descent,unary_negative_int # The trip count puts pypy's execution above the startup-subtraction floor, so # this ratio is a measurement. A ceiling far above the measurement would # disable the gate at both ends, the derived floor included. @@ -9,8 +8,8 @@ # UNARY_NEGATIVE in a hot loop lowers to the `unary_negative(value)` HLOp → -# `residual_call_r_r(unary_negative_fn, ListR[value])` through -# opcode_ops::unary_negative_value (mirroring UNARY_INVERT / UNARY_NOT). +# the canonical `inline_call_r_r` to the interpreter's `neg` body through +# opcode_ops::unary_negative_value (mirroring UNARY_INVERT). # Before the HLOp lowering the flow op `neg` reached the assembler with no # builder mapping and any `-x` in a JIT-compiled loop panicked. def main(): @@ -25,15 +24,12 @@ def main(): # UNARY_NEGATIVE on INT_MIN: -INT_MIN overflows the machine-int range, so # descr_neg (intobject.py:628) takes the long branch and returns 2**63 as a -# W_LongObject. The walker fold pins the operand with GUARD_VALUE and takes -# the _make_ovf2long tail, so the compiled loop must agree with the long result -# rather than wrapping back to INT_MIN. -# -# This is the operand the descent declines, which is why the header names both -# labels: `main` above is walked (`unary_negative_descent`) and this loop is -# folded (`unary_negative_int`). Without the fold the promoted W_LongObject -# stays a loop argument, `compare_op_long` keeps its bigint call in the body, -# and the loop runs 2x slower. +# W_LongObject. The codewriter inline-call walks that overflow arm of `neg`, +# so the compiled loop must agree with the long result rather than wrapping +# back to INT_MIN. The promoted long crosses the loop header as an argument +# and the comparison keeps its bigint call in the body; the retired +# `unary_negative_int` fold pinned the operand with GUARD_VALUE instead, and +# this loop ran 3x faster under it. def main_int_min(): m = -9223372036854775807 - 1 # INT_MIN as a machine int acc = 0 diff --git a/pyre/check.py b/pyre/check.py index 85fd1e2d098..a8fc01ababb 100644 --- a/pyre/check.py +++ b/pyre/check.py @@ -2544,6 +2544,11 @@ def spec_fold_census(binary, path, timeout_s, wasm=False): env["PYRE_WASM_SPEC_CENSUS" if wasm else "PYRE_FBW_SPEC_CENSUS"] = "1" _, _, code, err = run_timed([binary, path], timeout_s=timeout_s, env=env) if code != 0: + # A census panic is a runtime failure, not a missing fold. Preserve + # the evidence just as the ordinary benchmark failure path does. + if err: + print("\n─── spec-fold census stderr ───") + print(err.rstrip()) return None, code return {m.group(1): int(m.group(2)) for m in SPEC_CENSUS_FOLD_RE.finditer(err)}, 0 @@ -5727,7 +5732,14 @@ def leg_reads_local_llbc(backend): # headroom, and both derived floors -- 0.483x and 0.567x -- stay under # the narrowest readings of 1.25x and 1.83x, which the same subtraction # moved up rather than down. - chk.run_bench("fib_recursive", f"{B}/fib_recursive.py", 5, 2, 2.9, 2, 3.4) + # Runs 33692288311 and 33860926996 measured this branch after the + # manual arithmetic folds moved to generated interpreter descent: + # dynasm reached 3.8x and cranelift 4.1x while fib_recursive's exact + # loop/bridge/guard snapshot stayed unchanged. 4.4x and 4.8x are those + # cross-host highs plus 15%; their 0.733x/0.8x floors remain below the + # historical 0.9x low. The recovery is the canonical codewriter inline + # call to the selected arithmetic body, not restoring binary_op_int. + chk.run_bench("fib_recursive", f"{B}/fib_recursive.py", 5, 2, 4.4, 2, 4.8) chk.run_bench("nested_loop", f"{B}/nested_loop.py", 5, None, 2, None, 3) chk.run_bench("raise_catch", f"{B}/raise_catch_loop.py", 5, None, 1.5, None, 2.5) # Run 33363045302 measured spectral_norm at 0.4-1.4x on the healthy diff --git a/pyre/design.md b/pyre/design.md index 8afc6edb54d..90e6ecae240 100644 --- a/pyre/design.md +++ b/pyre/design.md @@ -589,16 +589,21 @@ compiled path is close to what it could be. ### 3.8 The fold layer: hand-written compensation for an opaque objspace -pyre records traces through 70 `try_walker_specialize_*` functions — 68 in -`jitcode_dispatch/specialize.rs`, one each in `residual_call.rs` -(`load_deref`) and `inline_call.rs` (`instance_next`) — 9,992 lines of body -inside `specialize.rs`'s 17,298, described by the 77 rows of +pyre records traces through 80 `try_walker_specialize_*` functions — 77 in +`jitcode_dispatch/specialize.rs`, one in `residual_call.rs` +(`load_deref`) and two in `inline_call.rs` (`instance_next`, +`generator_next`) — described by the 91 rows of `SPEC_FOLD_ROWS` (one fold can back several rows, and row-less folds exist). -Three of those rows are not folds at all: `subscr_tuple_descent`, -`unary_invert_descent` and `unary_negative_descent` are orthodox sub-walks -of `w_tuple_getitem`, `invert_inner` and `neg_inner`, carrying a row only so -they can be suppressed and A/B'd like the folds they replaced. Counting them -as debt overstates it by three. +Five of those rows are not folds at all: every label ending in `_descent` — +`subscr_tuple_descent`, `binary_op_descent`, `compare_op_descent`, +`builtin_len_descent` and `load_super_attr_descent` — is +an orthodox sub-walk through a `try_walker_orthodox_*` entry (8 of those +exist; the other three are the `list_append`/`list_pop` shapes), carrying a row only so it can +be suppressed and A/B'd like the fold it replaced. Counting them as debt +overstates it by five; the fold count is 86. The three unary descent rows +retired when `flatten` began emitting canonical codewriter `inline_call_r_r` +operations to `descroperation::pos`, `neg` and `invert`; their int and long +traces now enter those bodies without a residual-call gate. Nothing in this charter named that layer before 2026-08-26, which is itself the finding: it is the largest single adaptation in the tree. @@ -606,18 +611,21 @@ Re-derive every number here before citing it; this section has published two miscounts, and both survived because the recipe beside them did not run. Every command below is quoted as it must be typed. -* Rows — `spec_folds!` opens at `diag.rs:342` and closes at `:421`: - `sed -n '342,421p' pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs | rg -cF '=> ("'`. +* Rows — select the complete `spec_folds!` invocation by its symbol boundaries: + `sed -n '/^spec_folds! {/,/^}/p' pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs | rg -cF '=> ("'` + answers 91; replacing the final matcher with `rg -cF '_descent"'` answers + the 5 descent rows. A fixed line range is invalid here: the previous range + ended before the macro did and published a stale count. `-F` is load-bearing: without it the `(` is an unclosed regex group and `rg` exits 2 rather than counting. -* Definitions — `rg -c` reports one count *per file*, so it answers 68/1/1 - rather than 70. Sum the matches instead: +* Definitions — `rg -c` reports one count *per file*, so it answers 77/1/2 + rather than 80. Sum the matches instead: `rg -o 'fn try_walker_specialize_' pyre/ majit/ -g '*.rs' | wc -l`. -* Body lines — sum the brace-matched span of each `fn try_walker_specialize_*` - in `specialize.rs`; no one-liner does it. + The descent entries are a separate population: + `rg -o 'fn try_walker_orthodox_' pyre/ majit/ -g '*.rs' | wc -l` answers 8. * Corpus — `ls pyre/bench/synth/*.py | wc -l`. Non-recursive **on purpose**: - a recursive walk sweeps `_pending`, `foriter57` and `iter57` and answers - 533, over-counting by 46. Do not "fix" this to a `find`. + it answers 531. A recursive walk also sweeps archived subdirectories and + over-counts the active corpus. Do not "fix" this to a recursive `find`. `specialize.rs`'s own line count moved seven times in seven commits and is not a usable identifier for a tree. @@ -629,23 +637,26 @@ job is to *recognise* that residual and answer in its place. Upstream has no equivalent job, which means the obvious convergence — "retire the folds in favour of the ported optimizer" — is not available as stated. Group by group: -| group | n | nearest upstream | -|---|---|---| -| unbox → raw int/float/bigint arithmetic, compare, truth | 17 | `OptIntBounds`, `OptRewrite.optimize_INT_IS_TRUE`, `OptPure` — cleanup only | -| opaque builtin call → pure elidable call | 19 | `OptPure.optimize_CALL_PURE_I`; recognition is `jtransform._handle_math_sqrt_call` and `@jit.elidable`, not a pass | -| residual → `new_with_vtable` / `new_array` so it stays virtual | 10 | `OptVirtualize` removes such ops; the emitter is `MIFrame.opimpl_newlist` | -| guarded heap field / array / mapdict read and write | 19 | `OptHeap` CSEs them; the emitter is traced `LOAD_ATTR_caching` | -| type-identity shortcut | 0 | retired 2026-08-24; was `OptRewrite._optimize_oois_ooisnot` plus `Optimizer.constant_fold` — a real pass | -| frame / execution-context introspection | 6 | none at any layer; PyPy forces the virtualizable instead | -| function-object construction | 2 | none | - -The `n` column sums to 73, not to the 77 rows above it: the split was taken -when the layer had 73 rows and no one has re-derived it since. Re-derive it -by classifying every row, not by apportioning the difference — the two -miscounts this section already published both came from adjusting a -published number instead of recounting. - -Four groups have only downstream cleanup upstream, two have nothing at all, +| group | nearest upstream | +|---|---| +| unbox → raw int/float/bigint arithmetic, compare, truth, cast | `OptIntBounds`, `OptRewrite.optimize_INT_IS_TRUE`, `OptPure` — cleanup only | +| opaque builtin call → direct (mostly pure elidable) call | `OptPure.optimize_CALL_PURE_I`; recognition is `jtransform._handle_math_sqrt_call` and `@jit.elidable`, not a pass | +| residual → `new_with_vtable` / `new_array` so it stays virtual | `OptVirtualize` removes such ops; the emitter is `MIFrame.opimpl_newlist` | +| guarded heap field / array / mapdict read and write | `OptHeap` CSEs them; the emitter is traced `LOAD_ATTR_caching` | +| type-identity shortcut | retired 2026-08-24; was `OptRewrite._optimize_oois_ooisnot` plus `Optimizer.constant_fold` — a real pass | +| frame / execution-context introspection | none at any layer; PyPy forces the virtualizable instead | +| function-object construction | none | +| callee inlining (`instance_next`, `kwonly_defaults_inline`) | none as a pass; `MIFrame.opimpl_inline_call` reaches the callee by tracing into it | +| orthodox descent rows — not folds | the descent itself; the exact count is the symbol-derived 5 above | + +The groups are explanatory, not a second manually maintained census. Their +boundaries contain judgement calls (`set_add_method` may be read as a call or +a heap mutation, and the `super` rows mix virtual construction with frame +access), so attaching an independently edited `n` column made the table look +exact while letting it disagree with `SPEC_FOLD_ROWS`. The reproducible split +is 86 hand-written rows plus 5 orthodox descent rows, re-derived on 2026-09-03. + +Four groups have only downstream cleanup upstream, three have nothing at all, and exactly one has a counterpart that is a pass rather than a consumer. So the convergence target for this layer is **descent reach into the interpreter** — making the objspace walkable — and not the porting of an @@ -654,8 +665,8 @@ generation debt, but its stated repair is now the right one. **What was tried.** A gateway-wrapper pilot gave `math.sqrt` its own jitcode and a published `fnaddr`; the descent still declined on 433 transitive -blockers and retired zero folds. A census over the whole corpus — 481 -synthetic plus the macro benches, every one of the 73 rows observed — found +blockers and retired zero folds. A census over the whole corpus as it stood +on 2026-08-31 — 521 synthetic fixtures then, every row observed — found no fold with `consulted=0`, so the layer is not merely carrying dead arms. `load_deref` alone never fires, and naming each of its early returns shows why: all 38 declines across the 359 fixtures holding a nested function report @@ -664,9 +675,13 @@ That is a missing capability rather than dead weight — closure-callee inlining needs the fold, the fold needs constant cells, and constant cells need the inlining. -**What does not hold it in place.** 23 fixtures carry a `spec-folds=` header -and they name 44 distinct folds between them, so 29 of the 73 rows have no -fixture coupling at all. Retirement is blocked by reach, not by headers. +**What does not hold it in place.** 50 fixtures carry a `spec-folds=` header +and they name 61 distinct rows between them (3 of those descent rows; +`subscr_tuple_descent` and `load_super_attr_descent` are named by none), so +30 of the 91 rows have no fixture coupling at all +(`rg -o --no-filename --max-depth 1 'spec-folds=[^ ]+' pyre/bench/synth -g '*.py' | sed 's/spec-folds=//' | tr ',' '\n' | sort -u | wc -l`; +`-o` must be spelled without `-h`, which is `rg`'s help flag). Retirement +is blocked by reach, not by headers. **The bounded first step has been taken, and it does not settle the question.** The type-identity group — `builtin_type`, `builtin_isinstance`, @@ -675,11 +690,11 @@ question.** The type-identity group — `builtin_type`, `builtin_isinstance`, `[jit-stats]` counter and no wall clock. That is gate neutrality, not a descent: nothing in that change demonstrates the trace now carries the interpreter's own `abstract_isinstance_w` shape, and `isinstance` is still -recognised outside the layer, at `residual_call.rs:3416`, where it gates +recognised outside the layer by `try_specialize_isinstance_call`, where it gates replay-safety, carries no row, and the census cannot see it. Read that commit's message with care — it claims five retirements including `load_type_name_attr`, but its diff never touches that identifier and the -fold is live at `specialize.rs:3931`. +fold remains live in `try_walker_specialize_load_type_name_attr`. **Falsification.** A retirement counts against this entry only if the trace it leaves is the interpreter's own shape. `MAJIT_LOG=1`'s diff --git a/pyre/extra_tests/snippets/getattr_descriptor_wtf8.py b/pyre/extra_tests/snippets/getattr_descriptor_wtf8.py new file mode 100644 index 00000000000..792b9b3487a --- /dev/null +++ b/pyre/extra_tests/snippets/getattr_descriptor_wtf8.py @@ -0,0 +1,67 @@ +# pyre-check: gate=1 +# DescrOperation.getattr/get and W_Property.get: builtin getattr follows the +# same descriptor protocol for UTF-8 and lone-surrogate names. In particular, +# the constant-name route must retain the receiver's method binding and the +# type-version, instance-shadowing, and property-fget invalidation guards. + + +def read_pair(obj, method_name, property_name): + total = 0 + i = 0 + while i < 3000: + total += getattr(obj, method_name)() + total += getattr(obj, property_name) + i += 1 + return total + + +def method(self): + return 3 + + +def replacement(self): + return 7 + + +def getter(self): + return 5 + + +def replacement_getter(self): + return 11 + + +class Example: + pass + + +for method_name, property_name in [('method', 'value'), ('\udc81', '\udc82')]: + descriptor = property(getter) + setattr(Example, method_name, method) + setattr(Example, property_name, descriptor) + obj = Example() + assert read_pair(obj, method_name, property_name) == 3000 * 8 + + # A newly shadowing instance attribute must supersede the bound method. + setattr(obj, method_name, lambda: 13) + assert read_pair(obj, method_name, property_name) == 3000 * 18 + delattr(obj, method_name) + + setattr(Example, method_name, replacement) + assert read_pair(obj, method_name, property_name) == 3000 * 12 + + # Mutating the descriptor's fget does not replace the type-dict binding. + descriptor.__init__(replacement_getter) + assert read_pair(obj, method_name, property_name) == 3000 * 18 + assert getattr(obj, '\udcff_missing', 23) == 23 + + +class Override(Example): + def __getattribute__(self, name): + if name == '\udc81': + return lambda: 17 + return object.__getattribute__(self, name) + + +assert read_pair(Override(), '\udc81', '\udc82') == 3000 * 28 +print('ok') diff --git a/pyre/extra_tests/snippets/jit_is_op_bigint_identity.py b/pyre/extra_tests/snippets/jit_is_op_bigint_identity.py new file mode 100644 index 00000000000..790fc160664 --- /dev/null +++ b/pyre/extra_tests/snippets/jit_is_op_bigint_identity.py @@ -0,0 +1,53 @@ +# pyre-check: gate=1 +"""`is` between two BigInt-backed ints, in a loop long enough to compile. + +Whether two equal-valued bigints are one object is an interpreter's own +choice, so the oracle is the interpreter: the answer is taken once on a cold +path and the hot loop must agree with it every iteration. + +What can break that agreement is the layout: the `IS_OP` fold guards the +object's layout and then emits a pointer compare, which is only sound where +no equal-valued pair of that layout is `is`. A bigint has a layout of its +own, so a fold that reads only the machine-word integer layout as +value-comparing emits the pointer compare here and loses precisely the equal +half of the loop. +""" + +BIG = 1 << 70 +ROUNDS = 4000 + +# Cold, single-shot, and therefore never compiled: this is the interpreter's +# own answer for the pair the loop below builds. +EQUAL_VALUES_ARE_ONE_OBJECT = (BIG + 1) is (BIG + 1) + + +def equal_half(rounds): + other = BIG + 1 + hits = 0 + i = 0 + while i < rounds: + this = BIG + (i & 1) + if this is other: + hits += 1 + i += 1 + return hits + + +def unequal_half(rounds): + other = BIG + 1 + misses = 0 + i = 0 + while i < rounds: + this = BIG + (i & 1) + if this is not other: + misses += 1 + i += 1 + return misses + + +expected_hits = ROUNDS // 2 if EQUAL_VALUES_ARE_ONE_OBJECT else 0 +assert equal_half(ROUNDS) == expected_hits, (equal_half(ROUNDS), expected_hits) +assert unequal_half(ROUNDS) == ROUNDS - expected_hits, ( + unequal_half(ROUNDS), + ROUNDS - expected_hits, +) diff --git a/pyre/extra_tests/snippets/recursion.py b/pyre/extra_tests/snippets/recursion.py index 2d3b2205d68..9ab14dc32f2 100644 --- a/pyre/extra_tests/snippets/recursion.py +++ b/pyre/extra_tests/snippets/recursion.py @@ -11,3 +11,30 @@ class Foo(object): # Since the default __str__ implementation calls __repr__ and __repr__ is # actually __str__, str(foo) should raise a RecursionError. assert_raises(RecursionError, str, foo) + + +# A comparison override implemented natively re-enters the comparison operator +# without pushing a Python frame, so the frame-count limit never sees the +# cycle: binding `_operator.eq` as a bound method makes `c == c` call +# `_operator.eq(c, c)`, which is the same operator again. The native stack +# check is what has to answer. +import types +import _operator + + +class Cmp(object): + pass + + +cmp = Cmp() +Cmp.__eq__ = types.MethodType(_operator.eq, cmp) +assert_raises(RecursionError, lambda: cmp == cmp) + + +# `object.__ne__` calls the receiver's live `__eq__`, so binding it as that +# `__eq__` closes a cycle inside one native body, again without a Python frame. +class Ne(object): + __eq__ = object.__ne__ + + +assert_raises(RecursionError, lambda: Ne() == Ne()) diff --git a/pyre/gate-triage.md b/pyre/gate-triage.md index aa1400fc96b..433c83b27d3 100644 --- a/pyre/gate-triage.md +++ b/pyre/gate-triage.md @@ -270,12 +270,14 @@ defaults to zero. Both are measurement inputs, not runtime experiments. `PYRE_FBW_NO_SPECIALIZE` is the one entry here that changes behaviour rather than reporting it: its comma-separated selectors (or the reserved `all`) turn -off that many of the 77 trace-time specialization rows (the `spec_folds!` -invocation at `jitcode_dispatch/diag.rs:342-421`; count them there rather +off that many of the 91 trace-time specialization rows (the `spec_folds!` +invocation in `jitcode_dispatch/diag.rs`; count its complete symbol body rather than trusting this sentence), and an unset variable suppresses none. Not all -75 are hand-written: `subscr_tuple_descent`, `unary_invert_descent` and -`unary_negative_descent` name orthodox sub-walks of the interpreter's own -body, and a row is what lets one be suppressed and A/B'd like any other. It is a measurement instrument — suppressing a fold is how the descent +91 are hand-written: `subscr_tuple_descent`, `binary_op_descent`, +`compare_op_descent`, `builtin_len_descent` and `load_super_attr_descent` +name orthodox sub-walks of the +interpreter's own body, and a row is what lets one be suppressed and A/B'd +like any other. It is a measurement instrument — suppressing a fold is how the descent wall behind it is made to print — so it retires with the folds it selects, not before them. `PYRE_FBW_SPEC_CENSUS` in §6c is its read-only half: the per-fold diff --git a/pyre/pyre-interpreter/src/baseobjspace.rs b/pyre/pyre-interpreter/src/baseobjspace.rs index 4aaeebedbd6..10d6558794c 100644 --- a/pyre/pyre-interpreter/src/baseobjspace.rs +++ b/pyre/pyre-interpreter/src/baseobjspace.rs @@ -5842,14 +5842,16 @@ fn getattr_str_impl(obj: PyObjectRef, name: &str, call_getattr: bool, suppress: { let module_type = crate::typedef::gettypeobject(&pyre_object::MODULE_TYPE); let shadows_with_foreign_instance_dict = unsafe { - lookup_where(obj_type.as_ptr(), "__dict__").is_some_and(|(owner, _)| { - let layout = pyre_object::w_type_get_layout_ptr(owner); - !layout.is_null() - && std::ptr::eq( - (*(*layout).typedef).instance_type, - &pyre_object::pyobject::INSTANCE_TYPE, - ) - }) + lookup_where_with_method_cache(obj_type.as_ptr(), "__dict__").is_some_and( + |(owner, _)| { + let layout = pyre_object::w_type_get_layout_ptr(owner); + !layout.is_null() + && std::ptr::eq( + (*(*layout).typedef).instance_type, + &pyre_object::pyobject::INSTANCE_TYPE, + ) + }, + ) }; if !module_type.is_null() && unsafe { issubtype_w(obj_type.as_ptr(), module_type) } @@ -6344,11 +6346,13 @@ fn getattr_str_impl(obj: PyObjectRef, name: &str, call_getattr: bool, suppress: // protocol. A `W_ObjectObject` receiver reaches this arm only // through its own class, so the check is skipped for it. let owner_dispatches_getattribute = is_instance(obj) - || lookup_where(w_type, "__getattribute__").is_some_and(|(owner, found)| { - std::ptr::eq(found, slot) - && (pyre_object::w_type_is_heaptype(owner) - || pyre_object::w_type_dispatches_own_getattribute(owner)) - }); + || lookup_where_with_method_cache(w_type, "__getattribute__").is_some_and( + |(owner, found)| { + std::ptr::eq(found, slot) + && (pyre_object::w_type_is_heaptype(owner) + || pyre_object::w_type_dispatches_own_getattribute(owner)) + }, + ); if !owner_dispatches_getattribute { // descriptor.py `W_Super.getattribute`: a `super` // subclass which inherits the builtin slot still runs the @@ -6814,15 +6818,23 @@ unsafe fn getattr_surrogate(obj: PyObjectRef, w_name: PyObjectRef, name: &Wtf8) if pyre_object::is_exact_type(obj, &pyre_object::descriptor::SUPER_TYPE) { return super_getattribute(obj, w_name); } - if pyre_object::descriptor::is_super(obj) - && let Some(w_type) = crate::typedef::r#type(obj) + if let Some(w_type) = crate::typedef::r#type(obj) && let Some(slot) = getattribute_if_not_from_object(w_type.as_ptr()) { - let owned_by_heap_type = - lookup_where(w_type.as_ptr(), "__getattribute__").is_some_and(|(owner, found)| { - std::ptr::eq(found, slot) && pyre_object::w_type_is_heaptype(owner) - }); - if !owned_by_heap_type { + // DescrOperation._handle_getattribute calls the receiver's slot + // before the default descriptor lookup, irrespective of the + // attribute name's encoding. Keep getattr_str_impl's existing + // owner admission: builtin forwarding slots must not recurse + // through this object-space entry point. + let owner_dispatches_getattribute = is_instance(obj) + || lookup_where_with_method_cache(w_type.as_ptr(), "__getattribute__").is_some_and( + |(owner, found)| { + std::ptr::eq(found, slot) + && (pyre_object::w_type_is_heaptype(owner) + || pyre_object::w_type_dispatches_own_getattribute(owner)) + }, + ); + if !owner_dispatches_getattribute && pyre_object::descriptor::is_super(obj) { return match super_getattribute(obj, w_name) { Ok(value) => Ok(value), Err(err) if err.kind == PyErrorKind::AttributeError => { @@ -6831,12 +6843,19 @@ unsafe fn getattr_surrogate(obj: PyObjectRef, w_name: PyObjectRef, name: &Wtf8) Err(err) => Err(err), }; } - match get_and_call_function(slot, obj, w_type.as_ptr(), &[w_name]) { - Ok(value) => return Ok(value), - Err(err) if err.kind == PyErrorKind::AttributeError => { - return instance_getattr_hook_or_err_wtf8(w_type.as_ptr(), obj, w_name, err); + if owner_dispatches_getattribute { + match get_and_call_function(slot, obj, w_type.as_ptr(), &[w_name]) { + Ok(value) => return Ok(value), + Err(err) if err.kind == PyErrorKind::AttributeError => { + return instance_getattr_hook_or_err_wtf8( + w_type.as_ptr(), + obj, + w_name, + err, + ); + } + Err(err) => return Err(err), } - Err(err) => return Err(err), } } match object_getattribute_surrogate(obj, w_name, name) { @@ -8238,7 +8257,8 @@ pub(crate) fn object_getattr_miss(obj: PyObjectRef, name: &str, call_getattr: bo if !is_type(w_metaclass) { continue; } - if let Some((owner, descr)) = lookup_where(w_metaclass, name) { + if let Some((owner, descr)) = lookup_where_with_method_cache(w_metaclass, name) + { // A `__dict__` descriptor is a genuine metatype override // only when its owner is a proper metaclass (a subclass // of `type`). A plain base mixed into the metaclass MRO @@ -9898,8 +9918,21 @@ pub(crate) unsafe fn lookup_where_pair( w_type: PyObjectRef, name: &str, ) -> Option<(PyObjectRef, PyObjectRef)> { - let value = lookup_in_type_where_uncached(w_type, name)?; - let class = lookup_where_class_uncached(w_type, name)?; + if !majit_metainterp::jit::we_are_jitted() { + // Outside a trace there is no residual boundary to keep, so one raw + // walk answers both halves at once. + return lookup_where(w_type, name); + } + // Traced code boxes the name so both residuals carry thin pointers only, + // the way `lookup_in_type_uncached_split` does for the value half: the + // `&str` projections above cannot be published as residual targets, and + // an unpublished callee blocks every descent whose body reaches this arm. + let w_name = pyre_object::unicodeobject::box_str_constant(Wtf8::new(name)); + let value = _lookup_in_type_uncached(w_type, w_name); + if value.is_null() { + return None; + } + let class = _lookup_where_class_uncached(w_type, w_name); Some((class, value)) } @@ -9967,6 +10000,59 @@ pub(crate) unsafe fn lookup_in_type_wtf8_uncached( lookup_where_pair_wtf8_uncached(w_type, name).map(|(_src, value)| value) } +/// Residual-call twin of [`lookup_in_type_wtf8_uncached`] for traced code: +/// the name arrives as an interned, immortal str object (`box_str_constant`) +/// because the residual call ABI cannot pass a `&Wtf8`, and the result is a +/// raw pointer with null for `None`. The same shape as +/// [`_pure_lookup_where_with_method_cache`] next to the cached arm, without +/// the elidable marker: this arm serves types without a version tag. +/// +/// # Safety +/// `w_type` may be null or a non-type; `w_name` must be a live str object. +#[majit_macros::dont_look_inside] +pub(crate) unsafe fn _lookup_in_type_uncached( + w_type: *mut PyObject, + w_name: *mut PyObject, +) -> *mut PyObject { + let name = pyre_object::unicodeobject::w_str_get_wtf8(w_name); + lookup_in_type_wtf8_uncached(w_type, name).unwrap_or(std::ptr::null_mut()) +} + +/// Class-half twin of [`_lookup_in_type_uncached`] for traced code: the +/// same boxed-name, thin-pointer ABI over [`lookup_where_class_uncached`], +/// null for `None`. +/// +/// # Safety +/// `w_type` may be null or a non-type; `w_name` must be a live str object. +#[majit_macros::dont_look_inside] +pub(crate) unsafe fn _lookup_where_class_uncached( + w_type: *mut PyObject, + w_name: *mut PyObject, +) -> *mut PyObject { + let name = pyre_object::unicodeobject::w_str_get_wtf8(w_name); + match name.as_str() { + Ok(s) => lookup_where_class_uncached(w_type, s).unwrap_or(std::ptr::null_mut()), + Err(_) => lookup_where_pair_wtf8_uncached(w_type, name) + .map_or(std::ptr::null_mut(), |(src, _value)| src), + } +} + +/// The uncached arms of [`lookup_in_type_where_wtf8`]. The ordinary +/// interpreter passes its borrowed name straight through; traced code boxes +/// the name so the residual call carries thin pointers only, which keeps the +/// un-lowerable `&Wtf8` callee out of every gateway body that reaches a type +/// lookup (`descent_decline` scans a body for such callees regardless of +/// which arm executes). +#[inline] +unsafe fn lookup_in_type_uncached_split(w_type: PyObjectRef, name: &Wtf8) -> Option { + if !majit_metainterp::jit::we_are_jitted() { + return lookup_in_type_wtf8_uncached(w_type, name); + } + let w_name = pyre_object::unicodeobject::box_str_constant(name); + let v = _lookup_in_type_uncached(w_type, w_name); + if v.is_null() { None } else { Some(v) } +} + unsafe fn lookup_where_pair_wtf8( w_type: PyObjectRef, name: &Wtf8, @@ -10519,7 +10605,7 @@ pub(crate) unsafe fn lookup_in_type_where_wtf8( name: &Wtf8, ) -> Option { if w_type.is_null() || !is_type(w_type) { - return lookup_in_type_wtf8_uncached(w_type, name); + return lookup_in_type_uncached_split(w_type, name); } // typeobject.py:505 — `promote(self)`. let _ = majit_metainterp::jit::promote(w_type); @@ -10527,7 +10613,7 @@ pub(crate) unsafe fn lookup_in_type_where_wtf8( let version_tag = w_type_version_tag(w_type); if version_tag == 0 { // typeobject.py:507-509 — no version tag: uncacheable. - return lookup_in_type_wtf8_uncached(w_type, name); + return lookup_in_type_uncached_split(w_type, name); } if !majit_metainterp::jit::we_are_jitted() { let v = _cached_lookup_where_name(w_type, name, version_tag).1; @@ -11028,11 +11114,11 @@ pub enum TypeAttrBinding { /// /// # Safety /// `w_obj` must be a valid, non-null object pointer. -unsafe fn instance_dict_does_not_shadow(w_obj: PyObjectRef, name: &str) -> Option<()> { +unsafe fn instance_dict_does_not_shadow_wtf8(w_obj: PyObjectRef, name: &Wtf8) -> Option<()> { if is_instance(w_obj) { // Mapdict side storage: the entry lookup reads the map chain and // allocates nothing. - return crate::objspace::std::mapdict::instance_node_getdictvalue(w_obj, Wtf8::new(name)) + return crate::objspace::std::mapdict::instance_node_getdictvalue(w_obj, name) .is_none() .then_some(()); } @@ -11054,6 +11140,10 @@ unsafe fn instance_dict_does_not_shadow(w_obj: PyObjectRef, name: &str) -> Optio None } +unsafe fn instance_dict_does_not_shadow(w_obj: PyObjectRef, name: &str) -> Option<()> { + unsafe { instance_dict_does_not_shadow_wtf8(w_obj, Wtf8::new(name)) } +} + /// The `getattr(w_obj, name)` shape that reduces, purely, to /// `w_method_new(w_descr, w_obj, w_type)` — the bound method /// `object.__getattribute__` builds when the name resolves to a plain @@ -11085,6 +11175,19 @@ unsafe fn instance_dict_does_not_shadow(w_obj: PyObjectRef, name: &str) -> Optio pub unsafe fn bound_method_attr_fast_path( w_obj: PyObjectRef, name: &str, +) -> Option<(PyObjectRef, u64, PyObjectRef, bool)> { + unsafe { bound_method_attr_fast_path_wtf8(w_obj, Wtf8::new(name)) } +} + +/// WTF-8-preserving twin of [`bound_method_attr_fast_path`]. Attribute names +/// in PyPy are RPython strings and may contain lone surrogates; the JIT must +/// not turn that representational detail into an opaque-call boundary. +/// +/// # Safety +/// Same contract as [`bound_method_attr_fast_path`]. +pub unsafe fn bound_method_attr_fast_path_wtf8( + w_obj: PyObjectRef, + name: &Wtf8, ) -> Option<(PyObjectRef, u64, PyObjectRef, bool)> { if w_obj.is_null() { return None; @@ -11122,11 +11225,11 @@ pub unsafe fn bound_method_attr_fast_path( // only reachable one. let owes_shadow_guard = is_instance(w_obj); if owes_shadow_guard { - instance_dict_does_not_shadow(w_obj, name)?; + unsafe { instance_dict_does_not_shadow_wtf8(w_obj, name)? }; } else if !getdict_backing_native(w_obj).is_null() { return None; } - let w_descr = lookup_in_type(w_type, name)?; + let w_descr = unsafe { lookup_in_type_where_wtf8(w_type, name)? }; // The exact shape `get()` binds through `w_method_new`: a `function` or a // `method_descriptor`. Both take the SAME arm there (the // `FUNCTION_TYPE || METHOD_DESCRIPTOR_TYPE` test above `w_method_new`), diff --git a/pyre/pyre-interpreter/src/error.rs b/pyre/pyre-interpreter/src/error.rs index d2e454d0600..acb2c70e8a3 100644 --- a/pyre/pyre-interpreter/src/error.rs +++ b/pyre/pyre-interpreter/src/error.rs @@ -539,6 +539,24 @@ pub unsafe fn pyerror_type_error_to_exc_object( PyError::type_error(msg).to_exc_object() } +/// The zero-division twin of [`pyerror_type_error_to_exc_object`]. +/// +/// `int_floordiv` and `int_mod` both raise `PyError::zero_division` with the +/// literal `ZERO_DIVISION_MSG`. Fusing that constructor with exception +/// materialisation keeps the non-RPython Rust `PyError` aggregate out of the +/// generated operator JitCode, so the orthodox descent carries the same +/// `W_BaseException` value that PyPy's `OperationError` path exposes. +/// +/// # Safety +/// `w_msg` must be a live `W_UnicodeObject`. +#[majit_macros::dont_look_inside] +pub unsafe fn pyerror_zero_division_to_exc_object( + w_msg: *mut pyre_object::PyObject, +) -> *mut pyre_object::PyObject { + let msg = unsafe { pyre_object::unicodeobject::w_str_get_wtf8(w_msg) }.to_owned(); + PyError::zero_division(msg).to_exc_object() +} + impl PyError { /// Forward the up-to-three GC-managed references a `PyError` holds — the /// cached exception object and the lazy NameError/AttributeError name/obj diff --git a/pyre/pyre-interpreter/src/jit_fnaddr.rs b/pyre/pyre-interpreter/src/jit_fnaddr.rs index ef15d492512..f83a90be827 100644 --- a/pyre/pyre-interpreter/src/jit_fnaddr.rs +++ b/pyre/pyre-interpreter/src/jit_fnaddr.rs @@ -24,6 +24,8 @@ pub trait ResidualSlot {} /// such a function receives `1` for `Some(p)` and `0` for `None`, never `p`. /// That is a silent wrong value rather than a crash, which is why this trait /// is deliberately narrow. +/// This is only a size/bank check: a narrow integer result still needs a +/// typed widening bridge before a word-returning dispatcher may call it. pub trait ResidualRet {} impl ResidualRet for () {} @@ -100,6 +102,20 @@ extern "C" fn bh_code_unit_at(code: i64, index: i64) -> i64 { i64::from(crate::pyopcode::code_unit_at(code, index as usize)) } +/// `descr.py CallDescr.create_call_stub`: call with the actual RESULT type, +/// then cast to Signed. A raw `-> bool` target defines only the low byte of +/// the result on x86, while our residual dispatcher reads a whole word. +/// The policy macro cannot emit this bridge from the opaque `PyObjectRef` +/// alias spelling, so supply it at the source-only registry boundary. +extern "C" fn bh_w_type_issubtype(w_type: i64, cls: i64) -> i64 { + unsafe { + pyre_object::w_type_issubtype( + w_type as pyre_object::PyObjectRef, + cls as pyre_object::PyObjectRef, + ) as i64 + } +} + /// Publication helpers that check the signature instead of erasing it. /// /// Taking `*const ()` means every caller casts, and a cast accepts any @@ -938,6 +954,21 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { // ABI-UNSOUND: `Result<*mut PyObject, error::PyError>` does not fit one residual slot. push_abi_unsound_fnaddr(&mut entries, wrapper.path, wrapper.func as *const ()); } + // `BUILTIN_WRAPPER_DESCRIPTORS` does not exist on wasm32 + // (`linkme::distributed_slice` has no arm for `target_os = "unknown"`), + // so the loop above registers nothing there and + // `bytecode_for_address(__majit_wrap_builtin_len)` finds no jitcode: the + // builtin `len` gateway descent then declines before its spec gate. The + // address is used for the jitcode lookup only — the gateway body is + // descended, never residual-called — so register the one wrapper the + // descent recognises explicitly, the way this file registers every other + // wasm-reachable helper. + #[cfg(target_arch = "wasm32")] + push_abi_unsound_fnaddr( + &mut entries, + "pyre_interpreter::builtins::__majit_wrap_builtin_len", + crate::builtins::__majit_wrap_builtin_len as *const (), + ); // `type_object()` accessors are `dont_look_inside` (`majit-translate` // `front::llbc_hints` stamps them: the JIT residualizes the `OnceLock` body @@ -1341,6 +1372,25 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { "pyre_interpreter::jit_unary_positive_value", crate::opcode_ops::jit_unary_positive_value, ); + // Codewriter `inline_call_r_r` targets the object-space graph names, not + // the opcode residual wrappers. Bind each graph to its dedicated + // one-word C-ABI entry point so both recording-time descent and + // guard-failure blackholing execute the same interpreter operation. + cp1( + &mut entries, + "pyre_interpreter::objspace::descroperation::neg", + crate::opcode_ops::jit_descroperation_neg, + ); + cp1( + &mut entries, + "pyre_interpreter::objspace::descroperation::invert", + crate::opcode_ops::jit_descroperation_invert, + ); + cp1( + &mut entries, + "pyre_interpreter::objspace::descroperation::pos", + crate::opcode_ops::jit_descroperation_pos, + ); cpa2( &mut entries, "pyre_interpreter::opcode_ops::jit_getitem", @@ -1462,15 +1512,14 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { // `w_type_issubtype` is the MRO membership scan (`_issubtype`, // typeobject.py), run under the JIT inside `_pure_issubtype` // (`@elidable_promote`, typeobject.py:1657). Its `#[dont_look_inside]` - // residualises the call; bind the `-> bool` Rust `fn` directly by - // qualified path (2-pointer args, JIT-representable, no C-ABI bridge). - let w_type_issubtype: unsafe fn(pyre_object::PyObjectRef, pyre_object::PyObjectRef) -> bool = - pyre_object::w_type_issubtype; - upa2( + // residualises the call. `CallDescr.create_call_stub` calls a boolean + // RESULT before widening to Signed: bind the word-ABI bridge, never the + // raw Rust function whose upper result-register bits are undefined. + cpa2( &mut entries, "pyre_object::typeobject::w_type_issubtype", "pyre_object::w_type_issubtype", - w_type_issubtype, + bh_w_type_issubtype, ); // `lookup_exc_class_for_kind` reads the process-global `EXC_CLASS_BY_KIND` // registry the tracer cannot model; its residual call rides a C-ABI @@ -1866,6 +1915,26 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { "pyre_interpreter::_pure_getdictvalue_no_unwrapping", pure_getdictvalue_no_unwrapping, ); + // The uncached arm's thin-pointer twins (`lookup_where_pair` under the + // JIT): a boxed name in, a raw pointer (null for `None`) out. + let lookup_in_type_uncached: unsafe fn( + *mut pyre_object::PyObject, + *mut pyre_object::PyObject, + ) -> *mut pyre_object::PyObject = crate::baseobjspace::_lookup_in_type_uncached; + up2( + &mut entries, + "pyre_interpreter::baseobjspace::_lookup_in_type_uncached", + lookup_in_type_uncached, + ); + let lookup_where_class_uncached: unsafe fn( + *mut pyre_object::PyObject, + *mut pyre_object::PyObject, + ) -> *mut pyre_object::PyObject = crate::baseobjspace::_lookup_where_class_uncached; + up2( + &mut entries, + "pyre_interpreter::baseobjspace::_lookup_where_class_uncached", + lookup_where_class_uncached, + ); // #346: null-collapsing stable-alloc primitive residualised via // `#[dont_look_inside]`, keeping the thread-local GC hook dispatch out of // the trace. @@ -2513,6 +2582,23 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { "pyre_interpreter::objspace::descroperation::jit_bigint_lshift_int_int_result", crate::objspace::descroperation::jit_bigint_lshift_int_int_result, ); + // `_make_ovf2long`: the overflowed add, subtract, and multiply recover + // their exact result from the two machine words directly. + cp2( + &mut entries, + "pyre_interpreter::objspace::descroperation::jit_bigint_add_int_int", + crate::objspace::descroperation::jit_bigint_add_int_int, + ); + cp2( + &mut entries, + "pyre_interpreter::objspace::descroperation::jit_bigint_sub_int_int", + crate::objspace::descroperation::jit_bigint_sub_int_int, + ); + cp2( + &mut entries, + "pyre_interpreter::objspace::descroperation::jit_bigint_mul_int_int", + crate::objspace::descroperation::jit_bigint_mul_int_int, + ); // Unary rbigint operations each take one payload pointer. cp1( &mut entries, @@ -2988,6 +3074,14 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { "pyre_interpreter::pyerror_type_error_to_exc_object", pyerror_type_error_to_exc_object, ); + let pyerror_zero_division_to_exc_object: extern "C" fn(i64) -> i64 = + crate::error::__majit_call_target_pyerror_zero_division_to_exc_object; + cpa1( + &mut entries, + "pyre_interpreter::error::pyerror_zero_division_to_exc_object", + "pyre_interpreter::pyerror_zero_division_to_exc_object", + pyerror_zero_division_to_exc_object, + ); // `elidable_cannot_raise` subclass-range check; the trampoline widens its // one-word bool return by zero-extension. let ll_issubclass: extern "C" fn(i64, i64) -> i64 = @@ -4048,10 +4142,12 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { // `jtransform.py:576-577 rewrite_op_int_floordiv = // _do_builtin_call` (which resolves the helper through // `support.py` `_ll_2_int_mod` / `:255` `_ll_2_int_floordiv`). - // The C-trunc residual call below is what the trace path sees; - // the Python-floor `ll_int_py_*` helpers stay available for the - // future route-(b) emitter (Python-bytecode `int.py_mod` / - // `int.py_div` direct calls) under the dotted-name keys. + // The C-trunc residual call below is what a Rust `/` / `%` in a + // descended body sees. The Python-floor `ll_int_py_*` pair + // registered after it is route (b): `int_floordiv` / `int_mod` + // call the interpreter's `#[oopspec("int.py_div")]` twins, so the + // generated `//` / `%` descent records the same elidable + // `int.py_div` / `int.py_mod` call the hand fold did. // // `register_macro_helper_trace_fnaddr` strips the leading segment // from `full_path`; for a single-segment path (no `::`) the entire @@ -4096,6 +4192,16 @@ fn build_jit_trace_fnaddrs() -> (Vec<(&'static str, i64)>, Vec) { "_ll_2_int_mod", majit_metainterp::blackhole::_ll_2_int_mod, ); + p2( + &mut entries, + "pyre_interpreter::objspace::descroperation::ll_int_py_div", + crate::objspace::descroperation::ll_int_py_div, + ); + p2( + &mut entries, + "pyre_interpreter::objspace::descroperation::ll_int_py_mod", + crate::objspace::descroperation::ll_int_py_mod, + ); // `support.py _ll_1_cast_uint_to_float` / `_ll_1_cast_float_to_uint` // residual-call targets emitted by @@ -4988,6 +5094,50 @@ mod tests { try_pop_to(depth); } + #[test] + fn subtype_residual_registers_the_word_abi() { + // descr.py CallDescr.create_call_stub calls the actual RESULT type + // before casting to Signed. The trampoline implements that + // conversion for the word-returning residual ABI; a raw Rust bool + // function leaves the upper return-register bits undefined on x86. + let target: extern "C" fn(i64, i64) -> i64 = super::bh_w_type_issubtype; + let entries = jit_trace_fnaddrs(); + for path in [ + "pyre_object::typeobject::w_type_issubtype", + "pyre_object::w_type_issubtype", + ] { + assert_eq!( + entries + .iter() + .find(|(name, _)| *name == path) + .map(|(_, addr)| *addr), + Some(target as *const () as usize as i64), + "{path} must widen the bool before the residual reads a word", + ); + } + } + + #[test] + fn jit_trace_fnaddrs_covers_codewriter_unary_graphs_with_word_abi_bridges() { + let bindings: HashMap<&'static str, i64> = jit_trace_fnaddrs().into_iter().collect(); + for (path, expected) in [ + ( + "pyre_interpreter::objspace::descroperation::neg", + crate::opcode_ops::jit_descroperation_neg as *const () as usize as i64, + ), + ( + "pyre_interpreter::objspace::descroperation::invert", + crate::opcode_ops::jit_descroperation_invert as *const () as usize as i64, + ), + ( + "pyre_interpreter::objspace::descroperation::pos", + crate::opcode_ops::jit_descroperation_pos as *const () as usize as i64, + ), + ] { + assert_eq!(bindings.get(path), Some(&expected), "missing {path}"); + } + } + /// Two registered functions must never share an address. /// /// `pyre-jit-trace`'s `patch_constants_i_fnaddrs` rewrites residual-call @@ -5457,6 +5607,18 @@ mod tests { bindings["pyre_interpreter::pyerror_type_error_to_exc_object"], fused ); + + let zero_division: extern "C" fn(i64) -> i64 = + crate::error::__majit_call_target_pyerror_zero_division_to_exc_object; + let zero_division = zero_division as *const () as usize as i64; + assert_eq!( + bindings["pyre_interpreter::error::pyerror_zero_division_to_exc_object"], + zero_division + ); + assert_eq!( + bindings["pyre_interpreter::pyerror_zero_division_to_exc_object"], + zero_division + ); } #[test] diff --git a/pyre/pyre-interpreter/src/objspace/descroperation.rs b/pyre/pyre-interpreter/src/objspace/descroperation.rs index 527221d3732..8fc272763c1 100644 --- a/pyre/pyre-interpreter/src/objspace/descroperation.rs +++ b/pyre/pyre-interpreter/src/objspace/descroperation.rs @@ -72,19 +72,27 @@ fn bigint_mod_inverse(base: &BigInt, modulus: &BigInt) -> Result BigInt { - a + b +/// `rbigint.add_int_int_bigint_result`, the sum half of `_make_ovf2long` +/// (intobject.py:509-514). The overflow recovery takes two Signed words and +/// reaches the exact bigint sum in one elidable call, with no rbigint +/// allocated for either operand; the MIR front retargets this seam to +/// `jit_bigint_add_int_int` the way [`bigint_lshift_int_int_result`] is +/// retargeted for the shift arm. +#[majit_macros::jit_elidable] +fn bigint_add_int_int(a: i64, b: i64) -> BigInt { + BigInt::add_int_int_bigint_result(a, b) } -#[majit_macros::elidable] -fn bigint_sub(a: BigInt, b: BigInt) -> BigInt { - a - b +/// `rbigint.sub_int_int_bigint_result`; see [`bigint_add_int_int`]. +#[majit_macros::jit_elidable] +fn bigint_sub_int_int(a: i64, b: i64) -> BigInt { + BigInt::sub_int_int_bigint_result(a, b) } -#[majit_macros::elidable] -fn bigint_mul(a: BigInt, b: BigInt) -> BigInt { - a * b +/// `rbigint.mul_int_int_bigint_result`; see [`bigint_add_int_int`]. +#[majit_macros::jit_elidable] +fn bigint_mul_int_int(a: i64, b: i64) -> BigInt { + BigInt::mul_int_int_bigint_result(a, b) } /// Host spelling of the already-zero-checked quotient half of @@ -453,6 +461,47 @@ pub extern "C" fn jit_bigint_add(a: i64, b: i64) -> pyre_object::longobject::Jit } } +/// `rbigint.add_int_int_bigint_result` payload — the exact bigint sum of two +/// machine ints, with no rbigint built for either operand. Pointer-ABI form of +/// [`bigint_add_int_int`]; see [`jit_bigint_and`] for the result encoding. +#[majit_macros::elidable_or_memerror] +pub extern "C" fn jit_bigint_add_int_int( + a: i64, + b: i64, +) -> pyre_object::longobject::JitBigIntResult { + pyre_object::longobject::encode_jit_bigint_result( + pyre_object::longobject::alloc_bigint_nursery_collecting( + BigInt::add_int_int_bigint_result(a, b), + ), + ) +} + +/// `rbigint.sub_int_int_bigint_result` payload. See [`jit_bigint_add_int_int`]. +#[majit_macros::elidable_or_memerror] +pub extern "C" fn jit_bigint_sub_int_int( + a: i64, + b: i64, +) -> pyre_object::longobject::JitBigIntResult { + pyre_object::longobject::encode_jit_bigint_result( + pyre_object::longobject::alloc_bigint_nursery_collecting( + BigInt::sub_int_int_bigint_result(a, b), + ), + ) +} + +/// `rbigint.mul_int_int_bigint_result` payload. See [`jit_bigint_add_int_int`]. +#[majit_macros::elidable_or_memerror] +pub extern "C" fn jit_bigint_mul_int_int( + a: i64, + b: i64, +) -> pyre_object::longobject::JitBigIntResult { + pyre_object::longobject::encode_jit_bigint_result( + pyre_object::longobject::alloc_bigint_nursery_collecting( + BigInt::mul_int_int_bigint_result(a, b), + ), + ) +} + // ── BigInt/machine-int arithmetic residuals ───────────────────────── // // pypy/objspace/std/longobject.py:_make_generic_descr_binop and descr_sub @@ -828,7 +877,7 @@ unsafe fn int_add(a: PyObjectRef, b: PyObjectRef) -> PyResult { let vb = int_value(b); match va.checked_add(vb) { Some(r) => Ok(w_int_new(r)), - None => Ok(w_long_new(bigint_add(BigInt::from(va), BigInt::from(vb)))), + None => Ok(w_long_new(bigint_add_int_int(va, vb))), } } @@ -837,7 +886,7 @@ unsafe fn int_sub(a: PyObjectRef, b: PyObjectRef) -> PyResult { let vb = int_value(b); match va.checked_sub(vb) { Some(r) => Ok(w_int_new(r)), - None => Ok(w_long_new(bigint_sub(BigInt::from(va), BigInt::from(vb)))), + None => Ok(w_long_new(bigint_sub_int_int(va, vb))), } } @@ -846,7 +895,7 @@ unsafe fn int_mul(a: PyObjectRef, b: PyObjectRef) -> PyResult { let vb = int_value(b); match va.checked_mul(vb) { Some(r) => Ok(w_int_new(r)), - None => Ok(w_long_new(bigint_mul(BigInt::from(va), BigInt::from(vb)))), + None => Ok(w_long_new(bigint_mul_int_int(va, vb))), } } @@ -863,11 +912,7 @@ unsafe fn int_floordiv(a: PyObjectRef, b: PyObjectRef) -> PyResult { let vb = BigInt::from(vb); return Ok(bigint_result(bigint_floordiv_nonzero(&va, &vb))); } - let q = va / vb; - let r = va % vb; - // Adjust: if remainder is nonzero and signs of operands differ, subtract 1. - let q = if r != 0 && (r ^ vb) < 0 { q - 1 } else { q }; - Ok(w_int_new(q)) + Ok(w_int_new(ll_int_py_div(va, vb))) } unsafe fn int_mod(a: PyObjectRef, b: PyObjectRef) -> PyResult { @@ -883,9 +928,34 @@ unsafe fn int_mod(a: PyObjectRef, b: PyObjectRef) -> PyResult { let vb = BigInt::from(vb); return Ok(bigint_result(bigint_modulo_nonzero(&va, &vb))); } - let r = va % vb; - let r = if r != 0 && (r ^ vb) < 0 { r + vb } else { r }; - Ok(w_int_new(r)) + Ok(w_int_new(ll_int_py_mod(va, vb))) +} + +/// rint.py `ll_int_py_div`: the floor quotient, from the truncating machine +/// quotient plus a sign correction. The `int.py_div` oopspec makes the call +/// an elidable residual the optimizer strength-reduces for a constant +/// divisor (`rewrite.py optimize_call_int_py_div`), which is what the +/// generated `//` descent needs to match the retired hand fold's trace. +/// Both callers check `y != 0` and `x != INT_MIN || y != -1` first, so the +/// `wrapping_*` never wrap. +#[majit_macros::oopspec("int.py_div(x, y)")] +pub(crate) fn ll_int_py_div(x: i64, y: i64) -> i64 { + let r = x.wrapping_div(y); + let p = r.wrapping_mul(y); + let u = if y < 0 { + p.wrapping_sub(x) + } else { + x.wrapping_sub(p) + }; + r.wrapping_add(u >> (i64::BITS - 1)) +} + +/// rint.py `ll_int_py_mod`, the remainder companion of [`ll_int_py_div`]. +#[majit_macros::oopspec("int.py_mod(x, y)")] +pub(crate) fn ll_int_py_mod(x: i64, y: i64) -> i64 { + let r = x.wrapping_rem(y); + let u = if y < 0 { r.wrapping_neg() } else { r }; + r.wrapping_add(y & (u >> (i64::BITS - 1))) } // ── Long (BigInt) arithmetic operations ───────────────────────────── @@ -3164,6 +3234,18 @@ pub(crate) unsafe fn needs_seq_binop_dispatch( || dunder_overridden(b, rev, t) } +/// [`needs_seq_binop_dispatch`] behind the promoted exact-builtin test of +/// both operands; see [`needs_numeric_binop_dispatch_unless_exact`]. +#[inline] +unsafe fn needs_seq_binop_dispatch_unless_exact( + a: PyObjectRef, + b: PyObjectRef, + base: SeqBase, + op: BinopDunder, +) -> bool { + !both_exact_builtin_instances_promoted(a, b) && needs_seq_binop_dispatch(a, b, base, op) +} + /// `bytes`/`bytearray` analog of `needs_seq_binop_dispatch`. The two /// builtin types share one `+` branch, so each operand is judged against /// its own builtin base (`bytes` vs `bytearray`). `dont_look_inside` @@ -3179,6 +3261,17 @@ pub(crate) unsafe fn needs_bytes_binop_dispatch( bytes_operand_overrides(a, fwd, rev) || bytes_operand_overrides(b, fwd, rev) } +/// [`needs_bytes_binop_dispatch`] behind the promoted exact-builtin test of +/// both operands; see [`needs_numeric_binop_dispatch_unless_exact`]. +#[inline] +unsafe fn needs_bytes_binop_dispatch_unless_exact( + a: PyObjectRef, + b: PyObjectRef, + op: BinopDunder, +) -> bool { + !both_exact_builtin_instances_promoted(a, b) && needs_bytes_binop_dispatch(a, b, op) +} + /// True when `obj`'s type overrides `fwd`/`rev` relative to its builtin /// base (`bytes` or `bytearray`). Only reached from the residual /// `needs_bytes_binop_dispatch`, so the type-static loads never enter a @@ -3365,6 +3458,20 @@ pub(crate) unsafe fn needs_numeric_binop_dispatch( numeric_operand_overrides(a, fwd, rev) || numeric_operand_overrides(b, fwd, rev) } +/// [`needs_numeric_binop_dispatch`] behind the promoted exact-builtin test +/// of both operands. Two exact builtins override nothing, and that answer +/// is decided on the traced graph (see +/// [`is_exact_builtin_instance_promoted`]), so a traced binary operator on +/// two builtins records neither this probe nor the override arm it gates. +#[inline] +unsafe fn needs_numeric_binop_dispatch_unless_exact( + a: PyObjectRef, + b: PyObjectRef, + op: BinopDunder, +) -> bool { + !both_exact_builtin_instances_promoted(a, b) && needs_numeric_binop_dispatch(a, b, op) +} + /// Set/frozenset analogue of the numeric override gate. The storage fast /// paths below are valid for exact builtins, but a heap subclass must enter /// `_call_binop_impl` so its forward/reflected override and the reflected- @@ -3383,6 +3490,54 @@ pub(crate) unsafe fn needs_set_binop_dispatch(a: PyObjectRef, b: PyObjectRef) -> || (pyre_object::is_set_or_frozenset(b) && !is_exact_setlike(b)) } +/// [`needs_set_binop_dispatch`] behind the promoted exact-builtin test of +/// both operands; see [`needs_numeric_binop_dispatch_unless_exact`]. +#[inline] +unsafe fn needs_set_binop_dispatch_unless_exact(a: PyObjectRef, b: PyObjectRef) -> bool { + !both_exact_builtin_instances_promoted(a, b) && needs_set_binop_dispatch(a, b) +} + +/// [`pyre_object::is_exact_builtin_instance`] with the class word promoted: +/// `jit.promote(w_type)` as `W_TypeObject.lookup` spells it, so the trace +/// pins `w_class` with a `guard_value` and the test against the payload's +/// canonical class folds. The specialised arity-2 tuple payload types map to +/// the canonical `tuple` class, as `pyre_object::is_exact_tuple` does. A null +/// `w_class` (the read-only singletons) is exact by the same rule as the +/// unpromoted predicate. +#[inline] +unsafe fn is_exact_builtin_instance_promoted(a: PyObjectRef) -> bool { + if pyre_object::tagged_int::CAN_BE_TAGGED && pyre_object::tagged_int::is_tagged_int(a) { + return true; + } + if a.is_null() { + return false; + } + let w_class = majit_metainterp::jit::promote((*a).w_class); + if w_class.is_null() { + return true; + } + let ob_type = (*a).ob_type; + use pyre_object::specialisedtupleobject::{ + SPECIALISED_TUPLE_FF_TYPE, SPECIALISED_TUPLE_II_TYPE, SPECIALISED_TUPLE_OO_TYPE, + }; + let builtin_class = if std::ptr::eq(ob_type, &SPECIALISED_TUPLE_II_TYPE) + || std::ptr::eq(ob_type, &SPECIALISED_TUPLE_FF_TYPE) + || std::ptr::eq(ob_type, &SPECIALISED_TUPLE_OO_TYPE) + { + pyre_object::get_instantiate(&pyre_object::TUPLE_TYPE) + } else { + pyre_object::get_instantiate(&*ob_type) + }; + std::ptr::eq(w_class, builtin_class) +} + +/// Both operands of a binary operator are exact builtin instances, each +/// decided by [`is_exact_builtin_instance_promoted`]. +#[inline] +unsafe fn both_exact_builtin_instances_promoted(a: PyObjectRef, b: PyObjectRef) -> bool { + is_exact_builtin_instance_promoted(a) && is_exact_builtin_instance_promoted(b) +} + /// Unary analog: true when numeric operand `a` overrides the unary /// special named by `op` relative to its builtin base. #[majit_macros::dont_look_inside] @@ -3396,10 +3551,21 @@ pub(crate) unsafe fn needs_numeric_unaryop_dispatch(a: PyObjectRef, op: UnaryDun /// Call the overriding unary special on a numeric subclass operand before /// the Rust fast path. Returns `None` when `a` is an exact builtin /// numeric or does not override `op`, so the caller falls through. +/// +/// The exact-builtin answer is decided on the traced graph, ahead of the +/// `dont_look_inside` probe: `space.lookup(w_obj, name)` promotes the +/// receiver's type, and the promoted class is what makes the lookup fold at +/// trace time. pyre's Python-level class is `w_class`, so that is the word +/// promoted; the payload class comes from `ob_type`, which the codewriter +/// already turns into `guard_class`. With both pinned, the comparison is a +/// constant and an exact builtin operand never records the probe at all. unsafe fn try_numeric_unaryop_override( a: PyObjectRef, op: UnaryDunder, ) -> Result, PyError> { + if is_exact_builtin_instance_promoted(a) { + return Ok(None); + } if !needs_numeric_unaryop_dispatch(a, op) { return Ok(None); } @@ -3486,7 +3652,7 @@ pub fn add(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `+` and `+=` — one body, see [`binop_type_error`]. pub(crate) fn add_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::Add); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::Add); // A sequence left operand has no `nb_add` to offer, so the numeric // override runs the right operand's alone; the concat branches below // are `PyNumber_Add`'s `sq_concat` fall-through and already reach @@ -3516,7 +3682,7 @@ pub(crate) fn add_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> // descroperation.py:664 "unicode + string subclass" — a str // subclass overriding `__add__`/`__radd__` must reach the // reflected dispatch; otherwise concat directly. - if needs_seq_binop_dispatch(a, b, SeqBase::Str, BinopDunder::Add) + if needs_seq_binop_dispatch_unless_exact(a, b, SeqBase::Str, BinopDunder::Add) && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__add__", "__radd__")? { @@ -3540,7 +3706,7 @@ pub(crate) fn add_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> } } if is_list(a) && is_list(b) { - if needs_seq_binop_dispatch(a, b, SeqBase::List, BinopDunder::Add) + if needs_seq_binop_dispatch_unless_exact(a, b, SeqBase::List, BinopDunder::Add) && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__add__", "__radd__")? { @@ -3569,7 +3735,7 @@ pub(crate) fn add_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> } } if is_tuple(a) && is_tuple(b) { - if needs_seq_binop_dispatch(a, b, SeqBase::Tuple, BinopDunder::Add) + if needs_seq_binop_dispatch_unless_exact(a, b, SeqBase::Tuple, BinopDunder::Add) && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__add__", "__radd__")? { @@ -3597,7 +3763,7 @@ pub(crate) fn add_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> // Only a real bytes-like rhs can carry a subclass `__radd__`; // a memoryview cannot, so dispatch only when both are bytes-like. if pyre_object::bytesobject::is_bytes_like(b) - && needs_bytes_binop_dispatch(a, b, BinopDunder::Add) + && needs_bytes_binop_dispatch_unless_exact(a, b, BinopDunder::Add) && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__add__", "__radd__")? { @@ -3653,14 +3819,14 @@ pub fn sub(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `-` and `-=` — one body, see [`binop_type_error`]. pub(crate) fn sub_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let set_override = needs_set_binop_dispatch(a, b); + let set_override = needs_set_binop_dispatch_unless_exact(a, b); if set_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__sub__", "__rsub__")? { return Ok(result); } - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::Sub); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::Sub); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__sub__", "__rsub__")? @@ -3707,7 +3873,7 @@ pub fn mul(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `*` and `*=` — one body, see [`binop_type_error`]. pub(crate) fn mul_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::Mul); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::Mul); // As in [`binop_add_impl`]: a sequence operand carries no // `nb_multiply`, so only the other operand's runs here. if numeric_override @@ -3844,7 +4010,8 @@ pub fn floordiv(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `//` and `//=` — one body, see [`binop_type_error`]. pub(crate) fn floordiv_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::FloorDiv); + let numeric_override = + needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::FloorDiv); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__floordiv__", "__rfloordiv__")? @@ -3900,7 +4067,7 @@ pub fn mod_(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `%` and `%=` — one body, see [`binop_type_error`]. pub(crate) fn mod_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::Mod); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::Mod); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__mod__", "__rmod__")? @@ -3982,7 +4149,8 @@ pub fn truediv(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `/` and `/=` — one body, see [`binop_type_error`]. pub(crate) fn truediv_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::TrueDiv); + let numeric_override = + needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::TrueDiv); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__truediv__", "__rtruediv__")? @@ -4040,7 +4208,7 @@ pub(crate) fn truediv_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) /// fast paths nor `__pow__` / `__rpow__` produce a result. fn pow_binary(a: &mut PyObjectRef, b: &mut PyObjectRef) -> Result, PyError> { unsafe { - let numeric_override = needs_numeric_binop_dispatch(*a, *b, BinopDunder::Pow); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(*a, *b, BinopDunder::Pow); if numeric_override { if let Some(result) = try_dispatch_binary_special(a, b, "__pow__", "__rpow__")? { return Ok(Some(result)); @@ -4811,7 +4979,7 @@ pub fn pow3(mut base: PyObjectRef, mut exp: PyObjectRef, mut modulus: PyObjectRe pub fn divmod(mut a: PyObjectRef, mut b: PyObjectRef) -> PyResult { let numeric_override; unsafe { - numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::DivMod); + numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::DivMod); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__divmod__", "__rdivmod__")? @@ -4999,7 +5167,7 @@ pub fn lshift(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `<<` and `<<=` — one body, see [`binop_type_error`]. pub(crate) fn lshift_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::LShift); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::LShift); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__lshift__", "__rlshift__")? @@ -5033,7 +5201,7 @@ pub fn rshift(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `>>` and `>>=` — one body, see [`binop_type_error`]. pub(crate) fn rshift_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::RShift); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::RShift); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__rshift__", "__rrshift__")? @@ -5067,14 +5235,14 @@ pub fn and_(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `&` and `&=` — one body, see [`binop_type_error`]. pub(crate) fn and_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let set_override = needs_set_binop_dispatch(a, b); + let set_override = needs_set_binop_dispatch_unless_exact(a, b); if set_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__and__", "__rand__")? { return Ok(result); } - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::And); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::And); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__and__", "__rand__")? @@ -5157,8 +5325,8 @@ pub(crate) fn or_impl(a: PyObjectRef, b: PyObjectRef, symbol: &str) -> PyResult } }; unsafe { - let set_override = needs_set_binop_dispatch(a, b); - let numeric = needs_numeric_binop_dispatch(a, b, BinopDunder::Or); + let set_override = needs_set_binop_dispatch_unless_exact(a, b); + let numeric = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::Or); if set_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__or__", "__ror__")? { @@ -5245,14 +5413,14 @@ pub fn xor(a: PyObjectRef, b: PyObjectRef) -> PyResult { /// `^` and `^=` — one body, see [`binop_type_error`]. pub(crate) fn xor_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> PyResult { unsafe { - let set_override = needs_set_binop_dispatch(a, b); + let set_override = needs_set_binop_dispatch_unless_exact(a, b); if set_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__xor__", "__rxor__")? { return Ok(result); } - let numeric_override = needs_numeric_binop_dispatch(a, b, BinopDunder::Xor); + let numeric_override = needs_numeric_binop_dispatch_unless_exact(a, b, BinopDunder::Xor); if numeric_override && let Some(result) = try_dispatch_binary_special(&mut a, &mut b, "__xor__", "__rxor__")? @@ -5294,12 +5462,6 @@ pub(crate) fn xor_impl(mut a: PyObjectRef, mut b: PyObjectRef, symbol: &str) -> /// Comparison operation dispatch. pub fn compare(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { - // RPython inserts a stack check on this recursive object-space call. - // Container comparisons recurse without pushing a Python frame (for - // example two distinct self-referential lists), so keep the same guard - // explicitly in the Rust port and raise RecursionError before exhausting - // the native stack. - crate::stack_check::stack_check()?; // A builtin subclass overriding the comparison dunder dispatches the // override first (with reflected-subclass priority); exact builtins and // non-overriding subclasses fall through to the by-layout comparison slot, @@ -5311,15 +5473,30 @@ pub fn compare(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { // `is_exact_builtin_instance` before it looks anything up. Deciding // that here spares the pair a reverse-dunder resolution and two MRO // lookups, which is the whole of the probe for the commonest - // comparison there is. + // comparison there is. The decision is promoted + // (`both_exact_builtin_instances_promoted`), so a traced comparison + // of two exact builtins records neither the probe nor this test. // // Only the probe is skipped. The subtype ordering below still runs // for such a pair, because `bool` is a proper subclass of `int` and // both are exact builtin instances. - let pair_can_override = !pyre_object::is_exact_builtin_instance(a) - || !pyre_object::is_exact_builtin_instance(b); - if pair_can_override && let Some(result) = try_compare_override(a, b, op)? { - return Ok(result); + let pair_can_override = !both_exact_builtin_instances_promoted(a, b); + if pair_can_override { + // `transform.py insert_ll_stackcheck` puts a stack check on a block + // of every call-graph cycle. An override implemented natively + // re-enters `compare` without pushing a Python frame, so the frame + // limit never fires: `C.__eq__ = types.MethodType(_operator.eq, c)` + // closes the cycle through `call_function_impl_result`, which runs + // no check of its own. The guard sits on the arm that can reach + // one rather than at the head, because an exact-builtin pair + // answers `false` above on a promoted decision, so a traced + // `int < int` records neither the probe nor this check. The + // by-layout container cycle is covered separately, by the check in + // [`compare_slot_rest`]. + crate::stack_check::stack_check()?; + if let Some(result) = try_compare_override(a, b, op)? { + return Ok(result); + } } // PyPy `descroperation.py:_make_comparison_impl` swaps the operands // whenever the right-hand type is a proper subtype, before invoking @@ -5334,8 +5511,10 @@ pub fn compare(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { // original operator and operand order. if !is_instance(a) && !is_instance(b) - && let (Some(a_type), Some(b_type)) = - (crate::typedef::r#type(a), crate::typedef::r#type(b)) + // Two `let` chains, not one tuple pattern: a tuple of two + // `Option`s is an allocation on the walked path. + && let Some(a_type) = crate::typedef::r#type(a) + && let Some(b_type) = crate::typedef::r#type(b) // Spelled as a raw-pointer identity, the way this file spells every // other one. `NonNull`'s own `!=` goes through `NonNull::ne`, // which is not a call a trace can lower, and this test now sits on @@ -5356,6 +5535,11 @@ pub fn compare(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { /// builtin comparison instead of re-entering override dispatch (which would /// recurse). pub fn compare_slot(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { + // The machine-int arm stands alone so that this function has no loop: + // the codewriter looks inside a loop-free graph only + // (`policy.py look_inside_graph`), and a traced `int < int` must reach + // `int_lt` through here. Every other layout's comparison, several of + // which iterate, lives in [`compare_slot_rest`], a residual on the trace. unsafe { if is_int_like(a) && is_int_like(b) { return match op { @@ -5367,6 +5551,22 @@ pub fn compare_slot(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { CompareOp::Ne => int_ne(a, b), }; } + } + compare_slot_rest(a, b, op) +} + +/// [`compare_slot`] for every pair that is not two machine ints. +#[inline(never)] +fn compare_slot_rest(a: PyObjectRef, b: PyObjectRef, op: CompareOp) -> PyResult { + // RPython inserts a stack check on this recursive object-space call. + // Container comparisons recurse through [`compare`] without pushing a + // Python frame (for example two distinct self-referential lists), so keep + // the same guard explicitly in the Rust port and raise RecursionError + // before exhausting the native stack. It sits on the recursive arm: a + // machine-int pair never recurses, and the check would otherwise be one + // residual call on every traced `int < int`. + crate::stack_check::stack_check()?; + unsafe { // longobject.py `_make_descr_cmp` and intobject.py // `_make_descr_cmp`: both mixed orders call an rbigint.int_* method // on the long payload. The int-left order uses the reversed relation. @@ -5853,7 +6053,11 @@ impl CompareOp { } /// Unary positive (`+a`). - +/// +/// `inline(never)` is load-bearing: rustc otherwise folds this body into its +/// one-call wrapper and the codewriter never mints the graph named by +/// `flatten.rs build_orthodox_inline_call_r_r`. +#[inline(never)] pub fn pos(a: PyObjectRef) -> PyResult { unsafe { if let Some(result) = try_numeric_unaryop_override(a, UnaryDunder::Pos)? { @@ -5944,7 +6148,11 @@ fn bad_operand_type(descr: &str, a: PyObjectRef) -> PyError { } /// Unary negation. - +/// +/// `inline(never)` is load-bearing: rustc otherwise folds this body into its +/// one-call wrapper and the codewriter never mints the graph named by +/// `flatten.rs build_orthodox_inline_call_r_r`. +#[inline(never)] pub fn neg(a: PyObjectRef) -> PyResult { unsafe { if let Some(result) = try_numeric_unaryop_override(a, UnaryDunder::Neg)? { @@ -6024,7 +6232,11 @@ pub(crate) fn bool_invert_deprecation_text() -> PyObjectRef { } /// Unary bitwise inversion. - +/// +/// `inline(never)` is load-bearing: rustc otherwise folds this body into its +/// one-call wrapper and the codewriter never mints the graph named by +/// `flatten.rs build_orthodox_inline_call_r_r`. +#[inline(never)] pub fn invert(a: PyObjectRef) -> PyResult { unsafe { if let Some(result) = try_numeric_unaryop_override(a, UnaryDunder::Invert)? { diff --git a/pyre/pyre-interpreter/src/objspace/std/mapdict.rs b/pyre/pyre-interpreter/src/objspace/std/mapdict.rs index 9705b01f295..edb1d25072a 100644 --- a/pyre/pyre-interpreter/src/objspace/std/mapdict.rs +++ b/pyre/pyre-interpreter/src/objspace/std/mapdict.rs @@ -2227,6 +2227,13 @@ pub unsafe fn instance_dict_attr_fast_path( unsafe fn property_descr_fast_path( w_obj: PyObjectRef, name: &str, +) -> Option<(PyObjectRef, u64, PyObjectRef)> { + unsafe { property_descr_fast_path_wtf8(w_obj, Wtf8::new(name)) } +} + +unsafe fn property_descr_fast_path_wtf8( + w_obj: PyObjectRef, + name: &Wtf8, ) -> Option<(PyObjectRef, u64, PyObjectRef)> { let map = unsafe { mapdict_map_or_null(w_obj) }; if map.is_null() { @@ -2240,7 +2247,7 @@ unsafe fn property_descr_fast_path( if version_tag == 0 { return None; } - let w_descr = unsafe { crate::baseobjspace::lookup_in_type(w_type, name) }?; + let w_descr = unsafe { crate::baseobjspace::lookup_in_type_where_wtf8(w_type, name) }?; // Exact type: the fold calls `fget`/`fset` directly, which stands in for // `type(w_descr).__get__` only where that cannot have been overridden // (`descroperation.py:169-176`). A `property` subclass keeps the base @@ -2269,7 +2276,18 @@ pub unsafe fn property_get_fast_path( w_obj: PyObjectRef, name: &str, ) -> Option<(PyObjectRef, u64, PyObjectRef, PyObjectRef)> { - let (w_type, version_tag, w_descr) = unsafe { property_descr_fast_path(w_obj, name) }?; + unsafe { property_get_fast_path_wtf8(w_obj, Wtf8::new(name)) } +} + +/// WTF-8-preserving twin of [`property_get_fast_path`]. +/// +/// # Safety +/// Same contract as [`property_get_fast_path`]. +pub unsafe fn property_get_fast_path_wtf8( + w_obj: PyObjectRef, + name: &Wtf8, +) -> Option<(PyObjectRef, u64, PyObjectRef, PyObjectRef)> { + let (w_type, version_tag, w_descr) = unsafe { property_descr_fast_path_wtf8(w_obj, name) }?; if unsafe { crate::baseobjspace::getattribute_if_not_from_object(w_type) }.is_some() { return None; } diff --git a/pyre/pyre-interpreter/src/opcode_ops.rs b/pyre/pyre-interpreter/src/opcode_ops.rs index 0ea5ae7a354..9474f17ae70 100644 --- a/pyre/pyre-interpreter/src/opcode_ops.rs +++ b/pyre/pyre-interpreter/src/opcode_ops.rs @@ -162,6 +162,10 @@ fn operator_symbol(op: BinaryOperator) -> &'static str { } } +/// `inline(never)` is load-bearing: rustc otherwise folds this body into its +/// one-call wrapper and the codewriter never mints the graph a trace descends +/// (`specialize.rs try_walker_orthodox_binary_op`). +#[inline(never)] pub fn binary_value_from_tag( a: PyObjectRef, b: PyObjectRef, @@ -214,6 +218,9 @@ pub fn compare_value( compare(a, b, cmp_op) } +/// `inline(never)` for the same reason as [`binary_value_from_tag`]: the +/// codewriter must mint the graph a trace descends. +#[inline(never)] pub fn compare_value_from_tag( a: PyObjectRef, b: PyObjectRef, @@ -1090,6 +1097,30 @@ pub extern "C" fn jit_unary_positive_value(value: i64) -> i64 { } } +// `CallControl.get_jitcode` gives each inlined graph its own callable +// `JitCode.fnaddr`. The source graphs below are Rust `PyResult` functions, +// whose native ABI is not the one-word Ref ABI used by codewriter +// `inline_call_r_r`. Publish distinct C-ABI entry points for those graph +// paths, just as translation supplies callable addresses for RPython graphs. +// Keep these as separate functions from the opcode residual bridges: the +// fnaddr registry deliberately rejects unrelated path names sharing one +// address, because address-keyed runtime rebinding would otherwise be +// ambiguous. +#[inline(never)] +pub extern "C" fn jit_descroperation_neg(value: i64) -> i64 { + jit_unary_negative_value(value) +} + +#[inline(never)] +pub extern "C" fn jit_descroperation_invert(value: i64) -> i64 { + jit_unary_invert_value(value) +} + +#[inline(never)] +pub extern "C" fn jit_descroperation_pos(value: i64) -> i64 { + jit_unary_positive_value(value) +} + #[majit_macros::jit_may_force] pub extern "C" fn jit_getitem(obj: i64, index: i64) -> i64 { match getitem(obj as PyObjectRef, index as PyObjectRef) { diff --git a/pyre/pyre-interpreter/src/stack_check.rs b/pyre/pyre-interpreter/src/stack_check.rs index 334d5450986..799be680891 100644 --- a/pyre/pyre-interpreter/src/stack_check.rs +++ b/pyre/pyre-interpreter/src/stack_check.rs @@ -763,7 +763,7 @@ pub fn stack_check() -> Result<(), PyError> { // CPython 3.14 checks `py_recursion_remaining`, i.e. Python interpreter // depth, independently of native stack protection. pyre's matching // counter is bumped around every user-function call. - recursion_depth_check(crate::call::py_recursion_depth())?; + check_recursion_depth()?; let current = current_sp(); let end = PYRE_STACKTOOBIG.stack_end.load(Ordering::Relaxed); let length = PYRE_STACKTOOBIG.stack_length.load(Ordering::Relaxed); @@ -779,6 +779,17 @@ pub fn stack_check() -> Result<(), PyError> { Ok(()) } +/// Run only [`stack_check`]'s logical Python-activation-depth half. +/// +/// A compiled fragment already carries the backend's native-stack probe at +/// its entry. Its guard exit still has to reproduce `PyFrame.execute_frame`'s +/// check against `sys.getrecursionlimit()`, without paying for that native +/// probe a second time when it enters the portal runner. +#[inline] +pub fn check_recursion_depth() -> Result<(), PyError> { + recursion_depth_check(crate::call::py_recursion_depth()) +} + /// One-word residual-call ABI for [`stack_check`]. pub extern "C" fn stack_check_jit_abi() -> i64 { match stack_check() { diff --git a/pyre/pyre-interpreter/src/typedef.rs b/pyre/pyre-interpreter/src/typedef.rs index 69924fc343a..6f314f20fba 100644 --- a/pyre/pyre-interpreter/src/typedef.rs +++ b/pyre/pyre-interpreter/src/typedef.rs @@ -2099,8 +2099,12 @@ pub fn w_type() -> PyObjectRef { } pub fn gettypeobject(tp: &PyType) -> PyObjectRef { + // Spelled as a `match` rather than `map_or(PY_NULL, |p| p.as_ptr())`: + // the closure's environment lowers to a synthetic aggregate constructor + // with no host symbol, which every descent reaching this call stopped + // at. The niche `Option>` itself is one pointer word. match gettypefor(tp as *const PyType) { - Some(w_type) => w_type.as_ptr(), + Some(p) => p.as_ptr(), None => PY_NULL, } } @@ -21587,6 +21591,13 @@ fn init_object_type(ns: PyObjectRef) { "__ne__", |args| { crate::type_methods::arity_slot(args, 1)?; + // `transform.py insert_ll_stackcheck` puts a stack check on + // a block of every call-graph cycle. The lookup below + // reaches this same body whenever the receiver's `__eq__` + // is `object.__ne__` (`A.__eq__ = object.__ne__`), and it + // closes that cycle without pushing a Python frame, so the + // frame-count limit never sees it. + crate::stack_check::stack_check()?; // objectobject.py descr__ne__: look up and call the live // receiver's __eq__ descriptor, then invert that one // result. Running the full comparison dispatcher here diff --git a/pyre/pyre-jit-trace/build/prepass.rs b/pyre/pyre-jit-trace/build/prepass.rs index adfe61c76e2..216e9a3ad91 100644 --- a/pyre/pyre-jit-trace/build/prepass.rs +++ b/pyre/pyre-jit-trace/build/prepass.rs @@ -1035,6 +1035,23 @@ fn real_main() { )], ..Default::default() }, + // The bodies behind the opcode residuals the walker descends + // (`specialize.rs try_walker_orthodox_descent`). The portal + // reaches none of them in source: `call_spec.rs` classifies the + // eval loop's `binary_value` call as a residual, and the residual + // is lowered to `jit_binary_value_from_tag`, whose body this is. + helper_graphs: vec![ + majit_translate::CallPath::from_segments([ + "pyre_interpreter", + "opcode_ops", + "binary_value_from_tag", + ]), + majit_translate::CallPath::from_segments([ + "pyre_interpreter", + "opcode_ops", + "compare_value_from_tag", + ]), + ], jit_drivers: vec![ majit_translate::JitDriverSpec { portal: majit_translate::CallPath::from_segments(["eval", "eval_loop_jit"]), diff --git a/pyre/pyre-jit-trace/src/descr.rs b/pyre/pyre-jit-trace/src/descr.rs index ac754b58fa2..3407b467e28 100644 --- a/pyre/pyre-jit-trace/src/descr.rs +++ b/pyre/pyre-jit-trace/src/descr.rs @@ -33,6 +33,11 @@ use majit_ir::{ // (`info.py:203-206`). What the tags still buy is disjoint index ranges, // so two descr kinds cannot collide on one `HeapCache` key — a flat // counter would have to preserve that much and nothing else. +/// `symbolic.py WORD` — the target pointer width. Every `Type::Ref` +/// field is one pointer, so its descr width derives from the target +/// instead of spelling a 64-bit literal. +const WORD: usize = core::mem::size_of::(); + const FIELD_DESCR_TAG: u32 = 0x1000_0000; const ARRAY_DESCR_TAG: u32 = 0x2000_0000; const SIZE_DESCR_TAG: u32 = 0x3000_0000; @@ -958,6 +963,12 @@ fn build_object_descr_group_with_extra_gc_edges( }, ) .collect(); + debug_assert!( + specs + .iter() + .all(|s| s.field_type != Type::Ref || s.field_size == WORD), + "a Ref field is one pointer; size it WORD, not a literal" + ); let mut gc_edges: Vec> = vec![W_CLASS_FIELD_DESCR.clone()]; gc_edges.extend(extra_gc_edges.iter().cloned()); let group = majit_ir::descr::make_simple_descr_group_keyed_with_headerless( @@ -1052,6 +1063,12 @@ fn build_bare_gcstruct_descr_group( }, ) .collect(); + debug_assert!( + specs + .iter() + .all(|s| s.field_type != Type::Ref || s.field_size == WORD), + "a Ref field is one pointer; size it WORD, not a literal" + ); let group = majit_ir::descr::make_simple_descr_group_keyed_with_headerless( SIZE_DESCR_TAG | (obj_size as u32 & 0x0FFF_FFFF), obj_size, @@ -1098,7 +1115,7 @@ static STRINGBUILDER_DESCR_GROUP: LazyLock = LazyLock::new ( "current_buf", rb::STRINGBUILDER_CURRENT_BUF_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -1134,7 +1151,7 @@ static STRINGBUILDER_DESCR_GROUP: LazyLock = LazyLock::new ( "extra_pieces", rb::STRINGBUILDER_EXTRA_PIECES_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -1168,7 +1185,7 @@ static STRINGPIECE_DESCR_GROUP: LazyLock = LazyLock::new(| ( "buf", rb::STRINGPIECE_BUF_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -1177,7 +1194,7 @@ static STRINGPIECE_DESCR_GROUP: LazyLock = LazyLock::new(| ( "prev_piece", rb::STRINGPIECE_PREV_PIECE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -1232,7 +1249,7 @@ static W_FLOAT_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_dict", FLOAT_W_DICT_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -1241,7 +1258,7 @@ static W_FLOAT_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_slots", FLOAT_W_SLOTS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -1261,7 +1278,7 @@ static W_LONG_DESCR_GROUP: LazyLock = LazyLock::new(|| { &[( "value", pyre_object::longobject::LONG_VALUE_OFFSET, - 8, + WORD, // The `value` slot is a gc-pointer to the BigInt payload, so it // enters `gc_fielddescrs` (the boxing SetfieldGc emits the write // barrier). Immutable: a long's payload is set once at creation. @@ -1868,18 +1885,34 @@ static RANGE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "start", RANGE_START_OFFSET, - 8, + WORD, + Type::Ref, + false, + true, + false, + ), + ( + "stop", + RANGE_STOP_OFFSET, + WORD, + Type::Ref, + false, + true, + false, + ), + ( + "step", + RANGE_STEP_OFFSET, + WORD, Type::Ref, false, true, false, ), - ("stop", RANGE_STOP_OFFSET, 8, Type::Ref, false, true, false), - ("step", RANGE_STEP_OFFSET, 8, Type::Ref, false, true, false), ( "length", RANGE_LENGTH_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2051,7 +2084,7 @@ static FUNCTION_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2144,7 +2177,7 @@ static W_METHOD_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_function", METHOD_W_FUNCTION_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2153,7 +2186,7 @@ static W_METHOD_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_self", METHOD_W_SELF_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2162,7 +2195,7 @@ static W_METHOD_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_class", METHOD_W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2171,7 +2204,7 @@ static W_METHOD_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_module", METHOD_W_MODULE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2184,7 +2217,7 @@ static W_METHOD_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2287,7 +2320,7 @@ static W_OBJECT_MUTABLE_CELL_DESCR_GROUP: LazyLock = LazyL &[( "w_value", W_OBJECT_MUTABLE_CELL_GC_PTR_OFFSETS[0], - 8, + WORD, Type::Ref, false, false, @@ -2322,7 +2355,7 @@ static W_CELL_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "contents", core::mem::offset_of!(Cell, contents), - 8, + WORD, Type::Ref, false, false, @@ -2345,7 +2378,7 @@ static W_CELL_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2371,7 +2404,7 @@ static W_CELL_DESCR_GROUP: LazyLock = LazyLock::new(|| { /// inherited Python class has to be a proper virtual field of this group. static W_SUPER_DESCR_GROUP: LazyLock = LazyLock::new(|| { use pyre_object::descriptor::{W_SUPER_GC_TYPE_ID, W_SUPER_OBJECT_SIZE, W_Super}; - let field = |key, offset| (key, offset, 8, Type::Ref, false, false, false); + let field = |key, offset| (key, offset, WORD, Type::Ref, false, false, false); build_object_descr_group_with_def_path( W_SUPER_OBJECT_SIZE, W_SUPER_GC_TYPE_ID, @@ -2423,7 +2456,7 @@ static W_LIST_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "items", std::mem::offset_of!(W_ListObject, items), - 8, + WORD, Type::Ref, false, false, @@ -2483,7 +2516,7 @@ static W_LIST_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "int_items.block", std::mem::offset_of!(W_ListObject, int_items) + INT_ARRAY_BLOCK_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2492,7 +2525,7 @@ static W_LIST_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "float_items.block", std::mem::offset_of!(W_ListObject, float_items) + FLOAT_ARRAY_BLOCK_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2505,7 +2538,7 @@ static W_LIST_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2606,7 +2639,7 @@ static W_TUPLE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "wrappeditems", std::mem::offset_of!(W_TupleObject, wrappeditems), - 8, + WORD, Type::Ref, false, true, @@ -2615,7 +2648,7 @@ static W_TUPLE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2676,7 +2709,7 @@ static SPECIALISED_TUPLE_II_DESCR_GROUP: LazyLock = LazyLo ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2725,7 +2758,7 @@ static SPECIALISED_TUPLE_FF_DESCR_GROUP: LazyLock = LazyLo ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2756,7 +2789,7 @@ static SPECIALISED_TUPLE_OO_DESCR_GROUP: LazyLock = LazyLo ( "value0", SPECIALISED_TUPLE_OO_VALUE0_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2765,7 +2798,7 @@ static SPECIALISED_TUPLE_OO_DESCR_GROUP: LazyLock = LazyLo ( "value1", SPECIALISED_TUPLE_OO_VALUE1_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2774,7 +2807,7 @@ static SPECIALISED_TUPLE_OO_DESCR_GROUP: LazyLock = LazyLo ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -2948,7 +2981,7 @@ static W_SLICE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_start", SLICE_START_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2957,7 +2990,7 @@ static W_SLICE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_stop", SLICE_STOP_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -2966,7 +2999,7 @@ static W_SLICE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "w_step", SLICE_STEP_OFFSET, - 8, + WORD, Type::Ref, false, true, @@ -3015,7 +3048,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "locals_cells_stack_w", crate::frame_layout::PYFRAME_LOCALS_CELLS_STACK_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3049,7 +3082,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "pycode", crate::frame_layout::PYFRAME_PYCODE_OFFSET, - 8, + WORD, Type::Ref, true, false, @@ -3058,7 +3091,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.debugdata", crate::frame_layout::PYFRAME_DEBUGDATA_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3067,7 +3100,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.lastblock", crate::frame_layout::PYFRAME_LASTBLOCK_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3076,7 +3109,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.f_generator_nowref", crate::frame_layout::PYFRAME_F_GENERATOR_NOWREF_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3085,7 +3118,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.w_yielding_from", crate::frame_layout::PYFRAME_W_YIELDING_FROM_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3094,7 +3127,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.f_backref", crate::frame_layout::PYFRAME_F_BACKREF_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3103,7 +3136,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.w_builtin", crate::frame_layout::PYFRAME_W_BUILTIN_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -3393,16 +3426,14 @@ fn new_w_class_field_descr() -> Arc { // collide with the first value field, e.g. `W_IntObject.intval`). Arc::new(PyreFieldDescr { offset: pyre_object::pyobject::W_CLASS_OFFSET, - // `WORD` on paper — the field is a `*mut PyObject`, so 4 bytes on - // wasm32, and the build-time descr pool already sizes it that way - // (`call.rs get_type_flag` → `layout::target_word_size()`). Deriving it - // here to match makes `synth/exception_traceback_loop_forms` lose one - // iteration's `e.__traceback__` on the wasm backend, so the two - // universes stay deliberately out of step until that is understood. - // `state.rs materialize_virtual_object` keys its w_class branch off - // `field_size == size_of::<*mut PyObject>()`, a guard that therefore - // never fires on wasm32. - field_size: 8, + // One pointer, sized from the target: the build-time descr pool + // sizes this field by `layout::target_word_size()` (`call.rs + // get_type_flag`), and the two universes must agree for the + // canonical-descr bridge (`make_descr_from_bh`) and for + // `state.rs materialize_virtual_object`'s w_class branch, both of + // which compare widths. A fixed 8 would also overlap the first + // payload field on wasm32, where the header is 8 bytes. + field_size: WORD, field_type: Type::Ref, signed: false, immutable: false, @@ -4064,13 +4095,11 @@ pub fn type_version_tag_descr() -> DescrRef { /// One object per run for the identity reason [`W_CLASS_FIELD_DESCR`] /// documents — `heap.rs` keys its field cache on the `Arc` pointer, so a /// per-call descriptor would miss its own cache on every read. The size -/// follows the same descriptor's: 8 for a `PyObjectRef` on every target, not -/// the 4 bytes a wasm32 pointer occupies. `synth/type_name_attr_fold` reads -/// the same name under wasm as under both native backends. +/// follows the same descriptor's: one pointer, from the target. static TYPE_NAME_OBJ_FIELD_DESCR: LazyLock = LazyLock::new(|| { make_field_descr( core::mem::offset_of!(pyre_object::typeobject::W_TypeObject, w_name), - 8, + WORD, Type::Ref, false, ) @@ -4381,7 +4410,7 @@ static W_OBJECT_OBJECT_DESCR_GROUP: LazyLock = LazyLock::n ( "W_ObjectObject.storage", core::mem::offset_of!(pyre_object::W_ObjectObject, storage), - 8, + WORD, Type::Ref, false, false, @@ -4390,7 +4419,7 @@ static W_OBJECT_OBJECT_DESCR_GROUP: LazyLock = LazyLock::n ( "PyObject.w_class", pyre_object::pyobject::W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -4436,7 +4465,15 @@ fn build_native_user_mapdict_group( false, false, ), - ("storage", storage_offset, 8, Type::Ref, false, false, false), + ( + "storage", + storage_offset, + WORD, + Type::Ref, + false, + false, + false, + ), ], simple_name, def_path, @@ -5083,7 +5120,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_class", W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5092,7 +5129,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.args_w", EXC_ARGS_W_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5105,7 +5142,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_context", EXC_W_CONTEXT_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5120,7 +5157,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_cause", EXC_W_CAUSE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5129,7 +5166,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_traceback", EXC_W_TRACEBACK_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5138,7 +5175,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_object", EXC_W_OBJECT_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5147,7 +5184,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_start", EXC_W_START_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5156,7 +5193,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_end", EXC_W_END_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5165,7 +5202,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_reason", EXC_W_REASON_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5174,7 +5211,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_encoding", EXC_W_ENCODING_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5183,7 +5220,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_errno", EXC_W_ERRNO_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5192,7 +5229,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_winerror", EXC_W_WINERROR_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5201,7 +5238,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_strerror", EXC_W_STRERROR_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5210,7 +5247,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_filename", EXC_W_FILENAME_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5219,7 +5256,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_filename2", EXC_W_FILENAME2_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5228,7 +5265,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_code", EXC_W_CODE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5237,7 +5274,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_value", EXC_W_VALUE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5246,7 +5283,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_exc_name", EXC_W_NAME_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5255,7 +5292,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_attr_obj", EXC_W_ATTR_OBJ_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5264,7 +5301,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_import_path", EXC_W_IMPORT_PATH_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5273,7 +5310,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_import_name_from", EXC_W_IMPORT_NAME_FROM_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5282,7 +5319,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_import_msg", EXC_W_IMPORT_MSG_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5291,7 +5328,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_dict", EXC_W_DICT_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5300,7 +5337,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_weakreflifeline", EXC_W_WEAKREF_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5309,7 +5346,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_msg", EXC_W_SYNTAX_MSG_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5318,7 +5355,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_filename", EXC_W_SYNTAX_FILENAME_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5327,7 +5364,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_lineno", EXC_W_SYNTAX_LINENO_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5336,7 +5373,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_offset", EXC_W_SYNTAX_OFFSET_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5345,7 +5382,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_text", EXC_W_SYNTAX_TEXT_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5354,7 +5391,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_end_lineno", EXC_W_SYNTAX_END_LINENO_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5363,7 +5400,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_end_offset", EXC_W_SYNTAX_END_OFFSET_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5372,7 +5409,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_print_file_and_line", EXC_W_SYNTAX_PRINT_FILE_AND_LINE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5381,7 +5418,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_syntax_metadata", EXC_W_SYNTAX_METADATA_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5390,7 +5427,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_group_message", EXC_W_GROUP_MESSAGE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5399,7 +5436,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_group_exceptions", EXC_W_GROUP_EXCEPTIONS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5408,7 +5445,7 @@ fn build_w_exception_group(kind: ExcKind) -> PyreObjectDescrGroup { ( "W_BaseException.w_group_exceptions_repr", EXC_W_GROUP_EXCEPTIONS_REPR_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5541,7 +5578,7 @@ static PYTRACEBACK_DESCR_GROUP: LazyLock = LazyLock::new(| ( "PyTraceback.w_class", W_CLASS_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5550,7 +5587,7 @@ static PYTRACEBACK_DESCR_GROUP: LazyLock = LazyLock::new(| ( "PyTraceback.frame", PYTRACEBACK_FRAME_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5568,7 +5605,7 @@ static PYTRACEBACK_DESCR_GROUP: LazyLock = LazyLock::new(| ( "PyTraceback.w_next", PYTRACEBACK_W_NEXT_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -5586,7 +5623,7 @@ static PYTRACEBACK_DESCR_GROUP: LazyLock = LazyLock::new(| ( "PyTraceback.w_code", PYTRACEBACK_W_CODE_OFFSET, - 8, + WORD, Type::Ref, false, false, @@ -8214,15 +8251,12 @@ pub fn make_descr_from_bh(bh: &majit_translate::jitcode::BhDescr) -> DescrRef { // which would otherwise answer with the parent's own entry. // // Only when the two spellings describe the same memory access. - // `new_w_class_field_descr` hardcodes `field_size: 8` while the - // codewriter sizes a pointer field by `layout::target_word_size()` - // (`call.rs get_type_flag`), so on wasm32 the incoming descr is a - // 4-byte load and the canonical one an 8-byte load at the same - // offset. Merging them there would widen the read over four bytes - // of the adjacent payload. That size split is deliberate and - // documented at `new_w_class_field_descr`; until it is resolved the - // bridge declines rather than papering over it, leaving those - // targets exactly as they were before the bridge existed. + // Both universes size a pointer field from the target + // (`new_w_class_field_descr` uses `WORD`, the codewriter + // `layout::target_word_size()`), so the widths agree on every + // target; the check stays as the guard that keeps a + // differently-sized spelling from being widened onto the + // canonical descr. if name.as_str() == "w_class" && matches!( owner.as_str(), diff --git a/pyre/pyre-jit-trace/src/helpers.rs b/pyre/pyre-jit-trace/src/helpers.rs index f7e5512e4fc..8602bf9015f 100644 --- a/pyre/pyre-jit-trace/src/helpers.rs +++ b/pyre/pyre-jit-trace/src/helpers.rs @@ -623,6 +623,39 @@ pub(crate) fn emit_untag_int(ctx: &mut TraceCtx, obj: OpRef, value: i64) -> OpRe raw } +/// Teach the heap cache what a `NewWithVtable` determined about `new_op`. +/// +/// The class is known (`opimpl_new_with_vtable` → `class_now_known`; +/// `class_now_known` here takes the vtable address, since pyre tracks the +/// concrete class pointer where upstream only raises `HF_KNOWN_CLASS`), so a +/// later `guard_class` on `new_op` records nothing. The class word the +/// allocation writes from its size descr (`SizeDescr::w_class_obj`, the +/// same answer `OptVirtualize` folds the header read off a virtual with) is +/// cached as a recorded `setfield` would be: a later `getfield_gc_r` of +/// `w_class` on `new_op` yields the class constant and a `promote` of that +/// read records no guard. A size descr without a class word, or whose type +/// has no canonical class, seeds only the class. +pub fn note_class_word_after_new( + ctx: &mut TraceCtx, + new_op: OpRef, + size_descr: &majit_ir::DescrRef, +) { + let Some(size) = size_descr.as_size_descr() else { + return; + }; + ctx.heap_cache_mut() + .class_now_known(new_op, size.vtable() as i64); + let seed = size + .class_word_field() + .map(|field| field.index()) + .zip(size.w_class_obj()); + let Some((field_index, w_class)) = seed else { + return; + }; + let w_class = ctx.const_ref(w_class); + ctx.heapcache_getfield_now_known(new_op, field_index, w_class); +} + /// Emit inline W_Int creation (NewWithVtable + SetfieldGc). /// /// jtransform.py rewrite_op_setfield: setfield on typeptr is dropped @@ -637,8 +670,9 @@ pub fn emit_box_int_inline( // entirely ("ignore the operation completely -- instead, it's done by // 'new'"). rewrite.py handle_malloc_operation emits the vtable // setfield via fielddescr_vtable during GC rewrite of NEW_WITH_VTABLE. - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); + let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr.clone()); ctx.heap_cache_mut().new_object(new_op); + note_class_word_after_new(ctx, new_op, &size_descr); // Emit: SetfieldGc(v, intval, raw_int) let intval_idx = intval_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, raw_int], intval_descr); @@ -665,8 +699,9 @@ pub fn emit_box_long_inline( size_descr: majit_ir::DescrRef, value_descr: majit_ir::DescrRef, ) -> OpRef { - let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr); + let new_op = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], size_descr.clone()); ctx.heap_cache_mut().new_object(new_op); + note_class_word_after_new(ctx, new_op, &size_descr); let value_idx = value_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_op, bigint_ref], value_descr); ctx.heapcache_setfield_cached(new_op, value_idx, bigint_ref); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs index fba3d10d768..123fbbb41f0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs @@ -368,11 +368,8 @@ spec_folds! { // variant label site parent TruthInt => ("truth_int", "residual_call", "-"), TruthBool => ("truth_bool", "residual_call", "-"), - UnaryPositiveInt => ("unary_positive_int", "residual_call", "-"), - UnaryPositiveDescent => ("unary_positive_descent", "residual_call", "-"), - UnaryInvertDescent => ("unary_invert_descent", "residual_call", "-"), - UnaryNegativeDescent => ("unary_negative_descent", "residual_call", "-"), - UnaryNegativeInt => ("unary_negative_int", "residual_call", "-"), + BinaryOpDescent => ("binary_op_descent", "residual_call", "-"), + CompareOpDescent => ("compare_op_descent", "residual_call", "-"), StoreSubscr => ("store_subscr", "residual_call", "-"), Setslice => ("setslice", "residual_call", "-"), GetIter => ("get_iter", "residual_call", "-"), @@ -382,7 +379,7 @@ spec_folds! { Newtuple => ("newtuple", "residual_call", "-"), NewtupleObject => ("newtuple_object", "residual_call", "-"), Newlist => ("newlist", "residual_call", "-"), - BuiltinLen => ("builtin_len", "residual_call", "-"), + BuiltinLenDescent => ("builtin_len_descent", "inline_call", "-"), BuiltinIsinstance => ("builtin_isinstance", "residual_call", "-"), BuiltinDictGet => ("builtin_dict_get", "residual_call", "-"), BuiltinTypeGetattr => ("builtin_type_getattr", "residual_call", "-"), @@ -421,10 +418,10 @@ spec_folds! { LoadClassmethodAttr => ("load_classmethod_attr", "residual_call", "-"), LoadBoundMethodAttr => ("load_bound_method_attr", "residual_call", "-"), Subscr => ("subscr", "residual_call", "-"), - BinaryOpInt => ("binary_op_int", "residual_call", "-"), BinaryOpLongInt => ("binary_op_long_int", "residual_call", "-"), BinaryOpLongIntShift => ("binary_op_long_int_shift", "residual_call", "-"), BinaryOpLongIntDiv => ("binary_op_long_int_div", "residual_call", "-"), + BinaryOpIntZeroDiv => ("binary_op_int_zero_div", "residual_call", "-"), BinaryOpLongIntPow => ("binary_op_long_int_pow", "residual_call", "-"), BinaryOpLong => ("binary_op_long", "residual_call", "-"), TruedivOpLong => ("truediv_op_long", "residual_call", "-"), diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs index e35b7b066be..29a16bd1adb 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs @@ -11,6 +11,41 @@ use super::*; +/// The live pointer a ref operand carries, or `None` where the walk holds no +/// executable one. +/// +/// RPython's MIFrame register contains the FrontendOp itself, whose `.value` +/// is the concrete pointer `executor.do_getfield_gc_*` reads. Pyre also has a +/// typed register shadow, because some canonical and inlined helper arguments +/// are OpRefs allocated outside the active recorder and cannot be stamped +/// there; that shadow holds the same MIFrame box value, so it answers after +/// the ordinary OpRef carrier rather than instead of it. +/// +/// A null and an all-ones word are both rejected: null is no object, and +/// all-ones is the `vable_setfield` storage placeholder, which says "no +/// concrete known" rather than naming one. +fn concrete_ref_operand_ptr( + code: &[u8], + op: &DecodedOp, + operand_offset: usize, + obj: OpRef, + ctx: &WalkContext<'_, '_, Sym>, +) -> Option { + ctx.trace_ctx + .box_value(obj) + .and_then(|value| match value { + majit_ir::Value::Ref(reference) => Some(reference.0 as i64), + _ => None, + }) + .or_else( + || match read_ref_reg_concrete(code, op, operand_offset, ctx) { + ConcreteValue::Ref(reference) => Some(reference as usize as i64), + _ => None, + }, + ) + .filter(|&ptr| ptr != 0 && ptr != usize::MAX as i64) +} + /// `getarrayitem_gc_/rid>X` handler. Operand layout `rid>X`: /// 1B r-reg(array) + 1B i-reg(index) + 2B descr + 1B X-dst. /// @@ -485,24 +520,7 @@ pub(crate) fn getfield_gc_via_heapcache( let obj = read_ref_reg(code, op, 0, ctx)?; let descr = read_descr(code, op, 1, ctx)?; let descr_index = descr.index(); - // RPython's MIFrame register contains the FrontendOp itself, whose - // `.value` is the concrete pointer used by executor.do_getfield_gc_*. - // Pyre also has a typed register shadow because some canonical/inlined - // helper args are OpRefs allocated outside the active recorder and cannot - // be stamped there. Treat that shadow as the same MIFrame box value, - // after preferring the ordinary OpRef carrier. - let concrete_obj_ptr = ctx - .trace_ctx - .box_value(obj) - .and_then(|value| match value { - majit_ir::Value::Ref(reference) => Some(reference.0 as i64), - _ => None, - }) - .or_else(|| match read_ref_reg_concrete(code, op, 0, ctx) { - ConcreteValue::Ref(reference) => Some(reference as usize as i64), - _ => None, - }) - .filter(|&ptr| ptr != 0 && ptr != usize::MAX as i64); + let concrete_obj_ptr = concrete_ref_operand_ptr(code, op, 0, obj, ctx); // ConstPtr + always-pure fast path (pyjitpl.py): a constant // source through an immutable descr loads the field now and @@ -689,3 +707,62 @@ pub(crate) fn getfield_gc_via_heapcache( /// `virtualizable_gen.rs` pyre PyFrame static-field order /// `[last_instr, pycode, valuestackdepth, debugdata]`. pub(crate) const VABLE_CODE_FIELD_IDX: usize = 1; + +/// `pyjitpl.py opimpl_guard_class` for the `guard_class/r>X` op +/// `jtransform.rs rewrite_op_getfield` emits in place of a read of the +/// header's class word. The receiver's class is pinned with a `GuardClass` +/// unless the heapcache already knows it, and the op's result is that class +/// as a constant, in the bank (`dst_bank`) the replaced read was allocated +/// to. A constant receiver records no guard, as `generate_guard` does not. +/// +/// The class is read off the live receiver: the walker executes as it +/// records, so the concrete pointer is in the box value or the register +/// shadow. A receiver whose pointer neither carries is one this walk cannot +/// execute a header read for, which [`getfield_gc_via_heapcache`] would +/// have recorded symbolically; the class cannot be pinned symbolically, so +/// the walk declines the op. +pub(crate) fn guard_class_record( + code: &[u8], + op: &DecodedOp, + ctx: &mut WalkContext<'_, '_, Sym>, + dst_bank: char, +) -> Result<(DispatchOutcome, usize), DispatchError> { + let obj = read_ref_reg(code, op, 0, ctx)?; + let dst = code[op.pc + 2] as usize; + let concrete_obj_ptr = concrete_ref_operand_ptr(code, op, 0, obj, ctx); + let Some(obj_ptr) = concrete_obj_ptr else { + if fbw_debug_abort_enabled() { + eprintln!("[fbw-abort] GuardClassReceiverNotConcrete pc={}", op.pc); + } + return Err(DispatchError::UnsupportedOpname { + pc: op.pc, + key: "guard_class (receiver not concrete)", + }); + }; + // SAFETY: `obj_ptr` is a live object the walk is executing over; the + // class word is the first word of every `PyObject` header. + let cls = unsafe { (*(obj_ptr as *const pyre_object::PyObject)).ob_type } as i64; + ctx.trace_ctx + .profiler() + .count_ops(OpCode::GuardClass, majit_metainterp::counters::OPS); + if !obj.is_constant() { + walker_guard_class(ctx, op.pc, obj, cls)?; + } + match dst_bank { + 'i' => { + let result = ctx.trace_ctx.const_int(cls); + write_int_reg(ctx, op.pc, dst, result, ConcreteValue::Int(cls))?; + } + _ => { + let result = ctx.trace_ctx.const_ref(cls); + write_ref_reg( + ctx, + op.pc, + dst, + result, + ConcreteValue::Ref(cls as usize as pyre_object::PyObjectRef), + )?; + } + } + Ok((DispatchOutcome::Continue, op.next_pc)) +} diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 6b95d794cae..1e42cc70a4b 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -14,7 +14,8 @@ //! `dispatch_inline_call_*` per-shape dispatchers. The `inline_call_*` //! opname arms stay in `handle` (mod.rs) and call into these. -use majit_translate::codewriter::jitcode::DescentBlockerSummary; +use majit_translate::codewriter::jitcode::{DESCENT_ENTRY_LEN_SLOTS, DescentBlockerSummary}; +use rustpython_wtf8::Wtf8; use super::*; @@ -37,6 +38,57 @@ struct AttributeErrorInlineContext { name_concrete: pyre_object::PyObjectRef, } +/// Whether this concrete receiver takes PyPy's installed `__len__` shortcut +/// straight to a builtin layout's length body. +/// +/// `typedef.py use_special_method_shortcut('__len__')` replaces the generic +/// lookup in `descroperation.py _len` for builtin implementations. The +/// generated walk is execution-driven and therefore follows just that body, +/// while [`descent_blocker_summary`] deliberately joins the generic error and +/// override arms too. Those arms contain formatting helpers after apparent +/// effects and make the static pre-scan reject a clean exact-builtin walk. +/// +/// This predicate emits no IR and proves no later value. It only admits the +/// generated body; that body still records the payload-class and promoted +/// `w_class` guards which keep the recording-time shortcut valid at runtime. +unsafe fn exact_builtin_len_shortcut_receiver(obj: pyre_object::PyObjectRef) -> bool { + if obj.is_null() { + return false; + } + let ob_type = unsafe { (*obj).ob_type }; + let exact_w_class = if std::ptr::eq(ob_type, &pyre_object::pyobject::LIST_TYPE) { + // The admitted set is the strategies whose length read the generated + // body already lowers. `IntOrFloat` is `Integer`'s own read — + // `live_len` sends both to `ll_list_int_length` — so it is admitted on + // the same proof. Bytes/Ascii and the range strategies read a + // differently shaped nested storage, and exact `list` alone does not + // prove that lowering. + if !(unsafe { pyre_object::w_list_uses_int_storage(obj) } + || unsafe { pyre_object::w_list_uses_int_or_float_storage(obj) } + || unsafe { pyre_object::w_list_uses_float_storage(obj) } + || unsafe { pyre_object::w_list_uses_object_storage(obj) } + || unsafe { pyre_object::w_list_uses_empty_storage(obj) }) + { + return false; + } + pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::LIST_TYPE) + } else if std::ptr::eq(ob_type, &pyre_object::pyobject::TUPLE_TYPE) + || std::ptr::eq(ob_type, &pyre_object::pyobject::STR_TYPE) + || std::ptr::eq(ob_type, &pyre_object::bytesobject::BYTES_TYPE) + || std::ptr::eq(ob_type, &pyre_object::bytearrayobject::BYTEARRAY_TYPE) + || std::ptr::eq(ob_type, &pyre_object::setobject::SET_TYPE) + || std::ptr::eq(ob_type, &pyre_object::setobject::FROZENSET_TYPE) + || std::ptr::eq(ob_type, &pyre_object::functional::RANGE_TYPE) + { + pyre_object::pyobject::get_instantiate(unsafe { &*ob_type }) + } else if specialised_pair_kind(ob_type).is_some() { + pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::TUPLE_TYPE) + } else { + return false; + }; + std::ptr::eq(unsafe { (*obj).w_class }, exact_w_class) +} + /// Where an element of a `defs_w` tuple lives, and therefore what the trace /// emits to read one. `w_tuple_new` routes EVERY arity-2 tuple through /// `makespecialisedtuple2` (`specialisedtupleobject.py`), so a callee @@ -710,9 +762,10 @@ pub(crate) fn exception_string_override_straight_line(body_code: &[u8]) -> bool /// refuses the wrapper for the shape of its error path. /// /// [`summarize_descent_blockers`] therefore walks the body's control-flow graph -/// carrying one fact — whether an effect has been executed on some path to this -/// point — and reports the two kinds of blocker separately. Of the two, only -/// `blocker_after_effect` is a decline. +/// carrying whether an effect has executed plus the Int/array-length facts it +/// can prove. A known condition follows one successor exactly; a red +/// condition conservatively joins both. It reports the two kinds of blocker +/// separately, and only `blocker_after_effect` is a decline. /// /// The other decline is the scan admitting it did not read the whole body. /// Everything the walk concludes rests on having seen every path, so a body it @@ -733,14 +786,23 @@ pub(crate) fn exception_string_override_straight_line(body_code: &[u8]) -> bool /// `jit_metadata.json` carries. /// /// The answer is memoized on the jitcode itself, so the scan runs once per -/// body rather than once per call site. Only this entry point memoizes: the -/// summary a cycle produces belongs to the occurrence that opened it, not to -/// the body, so [`summarize_descent_blockers`] caches nothing. -fn descent_decline(jitcode_index: usize) -> Option { +/// (body, entry argument-array length) rather than once per call site. Only +/// this entry point memoizes: the summary a cycle produces belongs to the +/// occurrence that opened it, not to the body, so +/// [`summarize_descent_blockers`] caches nothing, and a body entered with more +/// than [`DESCENT_ENTRY_LEN_SLOTS`] arguments recomputes. +fn descent_decline( + jitcode_index: usize, + entry_array_lengths: &[(usize, usize)], +) -> Option { if !descent_unlowered_helper_scan_enabled() { return None; } - let summary = descent_blocker_summary(jitcode_index); + let summary = match entry_array_lengths { + [] => descent_blocker_summary(jitcode_index), + &[(0, entry_len)] => descent_blocker_summary_for_entry_len(jitcode_index, entry_len), + other => summarize_descent_blockers_with_entry(jitcode_index, &mut Vec::new(), other), + }; if summary.body_not_walked { return Some(DescentDecline::BodyNotWalked); } @@ -908,10 +970,36 @@ fn collect_descent_effect_aware_blockers( here.push((blocker, effect)); false }, + &mut residual_call_is_effect_free, + &mut switch_descr_targets, + &[], ); for (blocker, effect) in here { record_reachable_blocker(out, blocker, entry_effect || effect); } + let memo = descent_blocker_summary(jitcode_index); + // Per-jitcode summaries (and the decode feeding them) are debug-abort + // output; inline-diag alone keeps the recursive walk quiet. + if fbw_debug_abort_enabled() { + eprintln!( + "[builtin-inline-summary] jitcode={jitcode_index} may_effect={} free={:?} after={:?} not_walked={}", + memo.may_execute_effect, + memo.blocker_effect_free.map(|b| format!("{b:#x}")), + memo.blocker_after_effect.map(|b| format!("{b:#x}")), + memo.body_not_walked + ); + if let Some(pc) = memo.first_effect_pc { + let (opname, descr) = crate::jitcode_runtime::decode_op_at(jitcode.code.as_slice(), pc) + .map(|d| { + let descr = descr_operand_index(jitcode.code.as_slice(), &d); + (d.opname, descr) + }) + .unwrap_or(("?", None)); + eprintln!( + "[builtin-inline-first-effect] jitcode={jitcode_index} pc={pc} op={opname} descr={descr:?}" + ); + } + } for (callee, caller_effect) in callees { collect_descent_effect_aware_blockers(callee, entry_effect || caller_effect, visited, out); } @@ -1008,6 +1096,20 @@ fn body_not_walked() -> DescentBlockerSummary { } } +/// Memoizing entry point for [`summarize_descent_blockers`] with the entry +/// argument-array length a call site knows. +fn descent_blocker_summary_for_entry_len( + jitcode_index: usize, + entry_len: usize, +) -> DescentBlockerSummary { + let Some(jitcode) = crate::jitcode_runtime::get_jitcode_ref_by_index(jitcode_index) else { + return body_not_walked(); + }; + jitcode.descent_blocker_summary_for_entry_len(entry_len, || { + summarize_descent_blockers_with_entry(jitcode_index, &mut Vec::new(), &[(0, entry_len)]) + }) +} + /// Memoizing entry point for [`summarize_descent_blockers`]. fn descent_blocker_summary(jitcode_index: usize) -> DescentBlockerSummary { let Some(jitcode) = crate::jitcode_runtime::get_jitcode_ref_by_index(jitcode_index) else { @@ -1022,9 +1124,9 @@ fn descent_blocker_summary(jitcode_index: usize) -> DescentBlockerSummary { /// One reachable point of [`summarize_descent_blockers`]'s dataflow. /// -/// Both fields only ever lose information when two paths meet — `effect` goes -/// false to true and a disagreeing `known_i` slot goes to `None` — so the -/// worklist converges. +/// The carried facts only lose information when two paths meet — `effect` goes +/// false to true, disagreeing known-value slots go to `None`, and freshness +/// goes true to false — so the worklist converges. #[derive(Clone)] struct DescentPoint { /// Whether some path from the body entry to here executed an effect. @@ -1033,6 +1135,15 @@ struct DescentPoint { /// `allocate_callee_register_banks` uses: the slots at and above /// `num_regs_i` are pre-filled from `constants_i`. known_i: Vec>, + /// Known lengths of Ref-bank slots that hold GC arrays. Generated + /// `__majit_wrap_*` gateways receive their concrete argument-array length + /// from the call site; copies preserve it and any other Ref producer drops + /// it back to unknown. + known_array_len_r: Vec>, + /// Which Ref-bank slots hold an object this body itself allocated (a + /// `new*` result) on every path to here. A write into such an object is + /// not an effect: a rewind discards the allocation with it. + fresh_r: Vec, } /// Whether stepping `opname` applies an effect the walk would have to undo. @@ -1079,13 +1190,258 @@ fn label_operand_offset(argcodes: &str) -> Option { None } +/// Register operands are one byte, so this many Ref-bank slots cover any body. +const FRESH_SLOTS: usize = 256; + +/// Whether `op` is a heap write whose object operand -- the leading `r` +/// register of every `setfield_gc_*` / `setarrayitem_gc_*` / +/// `setinteriorfield_gc_*` -- holds an object allocated by this body. The +/// vable spellings write the frame, which is never fresh, and are not asked. +fn heap_write_into_fresh_object( + code: &[u8], + op: &crate::jitcode_runtime::DecodedOp, + fresh_r: &[bool], +) -> bool { + (op.opname.starts_with("setfield_gc") + || op.opname.starts_with("setarrayitem_gc") + || op.opname.starts_with("setinteriorfield_gc")) + && op.argcodes.starts_with('r') + && code + .get(op.pc + 1) + .is_some_and(|&obj| fresh_r.get(obj as usize).copied().unwrap_or(false)) +} + +/// The descr-pool index an op's first `d` operand names, read past the +/// register and varlist operands before it the way `decode_op_at` advances +/// over them: the funcbox and varlists of a `residual_call_*`, the key +/// register of a `switch/id`. +fn descr_operand_index(code: &[u8], op: &crate::jitcode_runtime::DecodedOp) -> Option { + let mut cursor = op.pc + 1; + for c in op.argcodes.chars() { + match c { + 'i' | 'c' | 'r' | 'f' => cursor += 1, + 'I' | 'R' | 'F' => cursor += 1 + *code.get(cursor)? as usize, + 'd' => { + return Some( + *code.get(cursor)? as usize | ((*code.get(cursor + 1)? as usize) << 8), + ); + } + _ => return None, + } + } + None +} + +/// Whether the residual call behind `descr_index` applies no effect the walk +/// would have to undo, by the call's own effectinfo. +/// +/// An elidable or loop-invariant callee writes no live heap. A +/// `not_in_trace` callee is exempt for a different reason: what has to hold +/// is not that the call left nothing behind, but that calling it again is +/// harmless, because a rolled-back walk is another trace attempt and the +/// region will be walked again. `rlib/jit.py not_in_trace` states the +/// contract that makes that true — the call is still made "by the jit tracing +/// and blackholing, but not by the final assembler", so a callee already owes +/// the same answer once per attempt whether or not any attempt is undone, and +/// the tracer alone decides how many attempts there are. pyre's one such +/// callee, `ensure_object_subclass_ranges_initialized`, is a `OnceLock` +/// initializer, which is that property in its strongest form. +/// +/// Anything else keeps the scan's conservative reading of an executed +/// residual call as effectful. +fn residual_call_is_effect_free(descr_index: usize) -> bool { + let descrs = crate::jitcode_runtime::descr_ref_table(); + let Some(descr) = descrs.at(descr_index) else { + return false; + }; + let Some(call) = descr.as_call_descr() else { + return false; + }; + let info = call.get_extra_info(); + info.oopspecindex == majit_ir::descr::OopSpecIndex::NotInTrace + || matches!( + info.extraeffect, + majit_ir::ExtraEffect::ElidableCannotRaise + | majit_ir::ExtraEffect::ElidableCanRaise + | majit_ir::ExtraEffect::ElidableOrMemoryError + | majit_ir::ExtraEffect::LoopInvariant + ) +} + +/// The arm targets of the `switch/id` whose descr is `descr_index`, in key +/// order, or `None` when the descr is not a switch table or a listed key +/// does not resolve to a target. +fn switch_descr_targets(descr_index: usize) -> Option> { + let descrs = crate::jitcode_runtime::descr_ref_table(); + let descr = descrs.at(descr_index)?; + let switch = descr.as_switch_descr()?; + // Every listed key must resolve: dropping one would narrow the successor + // set, and a region this scan does not traverse is a region it wrongly + // reports clean. An unresolved key therefore widens the whole answer to + // `None`. + switch + .const_keys_in_order() + .iter() + .map(|&key| switch.lookup(key)) + .collect() +} + +/// Evaluate the Int result of the small, side-effect-free opcode family the +/// blocker scan needs to decide a following conditional edge. +/// +/// This is the static counterpart of the corresponding `opimpl_*` methods in +/// `pyjitpl.py`: it does not invent a value for a red operand. It only folds +/// when every input is already known: an `i` operand names a slot whose value +/// the scan carries (a register it folded, or a constant one of the pool slots +/// above `num_regs_i` was seeded with), and a `c` operand is the value itself. +/// Unknown and overflow-sensitive operations stay unknown, so the caller keeps +/// both successors. +fn known_int_result( + code: &[u8], + op: &crate::jitcode_runtime::DecodedOp, + known_i: &[Option], + known_array_len_r: &[Option], +) -> Option { + let int = |offset: usize| { + code.get(op.pc + 1 + offset) + .and_then(|&slot| known_i.get(slot as usize)) + .copied() + .flatten() + }; + // A `c` argcode is `assembler.py emit_const(allow_short=True)`: the small + // ConstInt is written inline as one signed byte, so the byte is the source + // value and there is no pool slot to index. + let immediate = |offset: usize| code.get(op.pc + 1 + offset).map(|&byte| byte as i8 as i64); + let bool_word = |value: bool| i64::from(value); + + match op.key { + "arraylen_gc/rd>i" => code + .get(op.pc + 1) + .and_then(|&slot| known_array_len_r.get(slot as usize)) + .copied() + .flatten() + .and_then(|value| i64::try_from(value).ok()), + "int_copy/i>i" => int(0), + "int_copy/c>i" => immediate(0), + "int_add/ii>i" => Some(int(0)?.wrapping_add(int(1)?)), + "int_sub/ii>i" => Some(int(0)?.wrapping_sub(int(1)?)), + "int_mul/ii>i" => Some(int(0)?.wrapping_mul(int(1)?)), + "int_and/ii>i" => Some(int(0)? & int(1)?), + "int_or/ii>i" => Some(int(0)? | int(1)?), + "int_xor/ii>i" => Some(int(0)? ^ int(1)?), + "int_neg/i>i" => Some(int(0)?.wrapping_neg()), + "int_invert/i>i" => Some(!int(0)?), + "int_is_true/i>i" => Some(bool_word(int(0)? != 0)), + "int_lt/ii>i" => Some(bool_word(int(0)? < int(1)?)), + "int_le/ii>i" => Some(bool_word(int(0)? <= int(1)?)), + "int_eq/ii>i" => Some(bool_word(int(0)? == int(1)?)), + "int_ne/ii>i" => Some(bool_word(int(0)? != int(1)?)), + "int_gt/ii>i" => Some(bool_word(int(0)? > int(1)?)), + "int_ge/ii>i" => Some(bool_word(int(0)? >= int(1)?)), + _ => None, + } +} + +/// Whether a known Int condition makes this `goto_if_not*` take its label. +/// `None` is the deliberately conservative answer for a red condition or for +/// the pointer/float forms, whose value domains this scan does not carry. +fn known_goto_if_taken( + code: &[u8], + op: &crate::jitcode_runtime::DecodedOp, + known_i: &[Option], +) -> Option { + let int = |offset: usize| { + code.get(op.pc + 1 + offset) + .and_then(|&slot| known_i.get(slot as usize)) + .copied() + .flatten() + }; + let predicate = match op.key { + "goto_if_not/iL" | "goto_if_not_int_is_true/iL" => int(0)? != 0, + "goto_if_not_int_is_zero/iL" => int(0)? == 0, + "goto_if_not_int_lt/iiL" => int(0)? < int(1)?, + "goto_if_not_int_le/iiL" => int(0)? <= int(1)?, + "goto_if_not_int_eq/iiL" => int(0)? == int(1)?, + "goto_if_not_int_ne/iiL" => int(0)? != int(1)?, + "goto_if_not_int_gt/iiL" => int(0)? > int(1)?, + "goto_if_not_int_ge/iiL" => int(0)? >= int(1)?, + _ => return None, + }; + Some(!predicate) +} + /// Recursive worker of [`descent_blocker_summary`]. `seen` is the stack of /// jitcode indices currently being scanned. fn summarize_descent_blockers( jitcode_index: usize, seen: &mut Vec, +) -> DescentBlockerSummary { + let mut cycle_hits = 0; + summarize_descent_blockers_with_entry_inner(jitcode_index, seen, &[], &mut cycle_hits) +} + +/// [`summarize_descent_blockers`] for a callee, memoized on the callee's body +/// when the answer belongs to that body. +/// +/// A callee is scanned with no entry facts, so what it computes is the same +/// fact-free summary [`descent_blocker_summary`] holds — except under a cycle, +/// where a body already on the stack answers "executes an effect, names no +/// blocker" for the occurrence that opened the cycle rather than for the body. +/// A subtree that took that arm nowhere did not read the stack at all and its +/// answer is a property of the body alone. Without the memo a callee shared by +/// several paths is re-walked once per path, and the walk is transitive, so the +/// scan a single gateway pays grows with the shape of the graph beneath it. +fn summarize_descent_blockers_cached( + jitcode_index: usize, + seen: &mut Vec, + cycle_hits: &mut u64, +) -> DescentBlockerSummary { + if seen.contains(&jitcode_index) { + return summarize_descent_blockers_with_entry_inner(jitcode_index, seen, &[], cycle_hits); + } + let Some(jitcode) = crate::jitcode_runtime::get_jitcode_ref_by_index(jitcode_index) else { + return summarize_descent_blockers_with_entry_inner(jitcode_index, seen, &[], cycle_hits); + }; + if let Some(cached) = jitcode.descent_blocker_summary_if_computed() { + return cached; + } + let cycles_before = *cycle_hits; + let summary = summarize_descent_blockers_with_entry_inner(jitcode_index, seen, &[], cycle_hits); + if *cycle_hits == cycles_before { + jitcode.descent_blocker_summary(|| summary); + } + summary +} + +/// [`summarize_descent_blockers`] with concrete facts known at the entry of +/// the top body. Callee facts are not guessed: without mapping an +/// `inline_call_*` varlist into the callee banks, recursive bodies use their +/// ordinary conservative summaries. +fn summarize_descent_blockers_with_entry( + jitcode_index: usize, + seen: &mut Vec, + entry_array_lengths: &[(usize, usize)], +) -> DescentBlockerSummary { + // This memoization fence belongs to this analysis stack, not to the + // executing thread. Every recursive callee shares it; an independent + // analysis (including a reentrant one) must not change its value. + let mut cycle_hits = 0; + summarize_descent_blockers_with_entry_inner( + jitcode_index, + seen, + entry_array_lengths, + &mut cycle_hits, + ) +} + +fn summarize_descent_blockers_with_entry_inner( + jitcode_index: usize, + seen: &mut Vec, + entry_array_lengths: &[(usize, usize)], + cycle_hits: &mut u64, ) -> DescentBlockerSummary { if seen.contains(&jitcode_index) { + *cycle_hits += 1; return DescentBlockerSummary { may_execute_effect: true, ..DescentBlockerSummary::default() @@ -1111,16 +1467,20 @@ fn summarize_descent_blockers( }; seen.push(jitcode_index); let descrs = crate::jitcode_runtime::descr_ref_table(); - let summary = summarize_body_blockers( + let summary = summarize_body_blockers_with( jitcode.code.as_slice(), jitcode.num_regs_i(), jitcode.constants_i.as_slice(), - |descr_index| { + |descr_index, _caller_effect| { descrs .at(descr_index) .and_then(|descr| descr.as_jitcode_descr().map(|jc| jc.jitcode_index())) - .map(|callee| summarize_descent_blockers(callee, seen)) + .map(|callee| summarize_descent_blockers_cached(callee, seen, cycle_hits)) }, + &mut |_blocker, _effect| true, + &mut residual_call_is_effect_free, + &mut switch_descr_targets, + entry_array_lengths, ); seen.pop(); summary @@ -1132,6 +1492,11 @@ fn summarize_descent_blockers( /// own summary, or `None` when that operand names no JitCode. Splitting it out /// keeps the analysis testable on a hand-built body, which is the only way to /// state the two-kind split without an installed jitcode table. +/// +/// Test-only: production reaches the same dataflow through +/// [`summarize_body_blockers_with`], because every one of the hooks defaulted +/// here has a real production answer. +#[cfg(test)] pub(crate) fn summarize_body_blockers( code: &[u8], num_regs_i: usize, @@ -1144,6 +1509,9 @@ pub(crate) fn summarize_body_blockers( constants_i, |descr_index, _caller_effect| callee_summary(descr_index), &mut |_blocker, _effect| true, + &mut |_descr_index| false, + &mut |_descr_index| None, + &[], ) } @@ -1160,12 +1528,28 @@ pub(crate) fn summarize_body_blockers( /// `inline_call`, which the summary itself does not need — it folds the two /// cases through `blocker_after_effect.or(blocker_effect_free)` — but which a /// walk descending into the callee needs to classify what it finds there. -fn summarize_body_blockers_with( +/// +/// `call_effect_free` answers a `residual_call_*` op's descr index with +/// whether that call is exempt from the effectful reading +/// `descent_op_applies_effect` gives every residual call; the production +/// answer is [`residual_call_is_effect_free`]. +/// +/// `switch_targets` answers a `switch/id` op's descr index with the arm +/// targets its table holds ([`switch_descr_targets`]); `None` widens the +/// successors to every instruction start. +/// +/// `entry_array_lengths` seeds concrete GC-array lengths known at this call +/// site. Production uses it for the generated builtin wrapper's sole `r0` +/// argument; tests and memoized whole-body summaries pass an empty slice. +pub(crate) fn summarize_body_blockers_with( code: &[u8], num_regs_i: usize, constants_i: &[i64], mut callee_summary: impl FnMut(usize, bool) -> Option, on_blocker: &mut dyn FnMut(i64, bool) -> bool, + call_effect_free: &mut dyn FnMut(usize) -> bool, + switch_targets: &mut dyn FnMut(usize) -> Option>, + entry_array_lengths: &[(usize, usize)], ) -> DescentBlockerSummary { let mut summary = DescentBlockerSummary::default(); @@ -1206,6 +1590,12 @@ fn summarize_body_blockers_with( // `residual_call` in a handler reads its funcbox from, and a funcbox that // reads unknown is a blocker the scan does not report. let handler_known = entry_known.clone(); + let mut entry_array_len_r = vec![None; FRESH_SLOTS]; + for &(slot, len) in entry_array_lengths { + if let Some(value) = entry_array_len_r.get_mut(slot) { + *value = Some(len); + } + } let mut points: std::collections::HashMap = std::collections::HashMap::new(); points.insert( @@ -1213,6 +1603,8 @@ fn summarize_body_blockers_with( DescentPoint { effect: false, known_i: entry_known, + known_array_len_r: entry_array_len_r, + fresh_r: vec![false; FRESH_SLOTS], }, ); let mut work = std::collections::VecDeque::from([0usize]); @@ -1241,6 +1633,22 @@ fn summarize_body_blockers_with( widened = true; } } + for (slot, incoming) in existing + .known_array_len_r + .iter_mut() + .zip(&state.known_array_len_r) + { + if *slot != *incoming && slot.is_some() { + *slot = None; + widened = true; + } + } + for (slot, incoming) in existing.fresh_r.iter_mut().zip(&state.fresh_r) { + if *slot && !*incoming { + *slot = false; + widened = true; + } + } if widened { $work.push_back(target); } @@ -1263,6 +1671,8 @@ fn summarize_body_blockers_with( }; let mut effect = point.effect; let mut known_i = point.known_i; + let mut known_array_len_r = point.known_array_len_r; + let mut fresh_r = point.fresh_r; if d.opname.starts_with("residual_call") { // Every `residual_call_*` argcode string opens with the `i` funcbox @@ -1302,6 +1712,7 @@ fn summarize_body_blockers_with( } if callee.may_execute_effect { effect = true; + summary.first_effect_pc.get_or_insert(d.pc); } // The descent enters this callee, so a region of it the callee's // own scan could not read is a region of this descent. @@ -1316,30 +1727,59 @@ fn summarize_body_blockers_with( // region of this descent the scan did not walk. summary.body_not_walked = true; effect = true; + summary.first_effect_pc.get_or_insert(d.pc); } } - if descent_op_applies_effect(d.opname) { + if descent_op_applies_effect(d.opname) + && !(d.opname.starts_with("residual_call") + && descr_operand_index(code, &d).is_some_and(|index| call_effect_free(index))) + && !heap_write_into_fresh_object(code, &d, &fresh_r) + { effect = true; + summary.first_effect_pc.get_or_insert(d.pc); } if effect { summary.may_execute_effect = true; } // An op writes at most one register, named by the argcode suffix after - // `>` and encoded as the instruction's last byte. Only `int_copy/i>i` - // carries a known value forward; every other Int-bank write makes its - // destination unknown again. + // `>` and encoded as the instruction's last byte. Carry every Int + // result whose inputs are already known; a red input or an unmodelled + // operation clears the destination back to unknown. if d.argcodes .split_once('>') .is_some_and(|(_, dst)| dst == "i") && let Some(&dst) = code.get(d.next_pc.wrapping_sub(1)) { - let carried = (d.key == "int_copy/i>i") + let carried = known_int_result(code, &d, &known_i, &known_array_len_r); + if let Some(slot) = known_i.get_mut(dst as usize) { + *slot = carried; + } + } + // A Ref-bank write is fresh only when a `new*` op produced it; any + // other producer -- a field read, a call result, a copy -- may name + // live heap. + if d.argcodes + .split_once('>') + .is_some_and(|(_, dst)| dst == "r") + && let Some(&dst) = code.get(d.next_pc.wrapping_sub(1)) + && let Some(slot) = fresh_r.get_mut(dst as usize) + { + *slot = d.opname.starts_with("new"); + } + // The wrapper-argument array can move between Ref colors before its + // length check. Only a plain ref copy preserves that entry fact. + if d.argcodes + .split_once('>') + .is_some_and(|(_, dst)| dst == "r") + && let Some(&dst) = code.get(d.next_pc.wrapping_sub(1)) + { + let carried = (d.key == "ref_copy/r>r") .then(|| code.get(d.pc + 1)) .flatten() - .and_then(|&src| known_i.get(src as usize).copied()) + .and_then(|&src| known_array_len_r.get(src as usize).copied()) .flatten(); - if let Some(slot) = known_i.get_mut(dst as usize) { + if let Some(slot) = known_array_len_r.get_mut(dst as usize) { *slot = carried; } } @@ -1347,6 +1787,8 @@ fn summarize_body_blockers_with( let state = DescentPoint { effect, known_i: known_i.clone(), + known_array_len_r: known_array_len_r.clone(), + fresh_r: fresh_r.clone(), }; let label = label_operand_offset(d.argcodes).map(|off| read_label(code, &d, off)); match d.opname { @@ -1376,6 +1818,8 @@ fn summarize_body_blockers_with( DescentPoint { effect: true, known_i: handler_known.clone(), + known_array_len_r: vec![None; FRESH_SLOTS], + fresh_r: vec![false; FRESH_SLOTS], } ); } @@ -1385,12 +1829,20 @@ fn summarize_body_blockers_with( push!(points, work, target, state); } } - name if name.starts_with("goto_if") => { - if let Some(target) = label { - push!(points, work, target, state.clone()); + name if name.starts_with("goto_if") => match known_goto_if_taken(code, &d, &known_i) { + Some(true) => { + if let Some(target) = label { + push!(points, work, target, state); + } } - push!(points, work, d.next_pc, state); - } + Some(false) => push!(points, work, d.next_pc, state), + None => { + if let Some(target) = label { + push!(points, work, target, state.clone()); + } + push!(points, work, d.next_pc, state); + } + }, "catch_exception" => { // The exception this handler catches can be raised by any op in // the region it protects, so the handler is entered with the @@ -1405,6 +1857,8 @@ fn summarize_body_blockers_with( DescentPoint { effect: true, known_i: handler_known.clone(), + known_array_len_r: vec![None; FRESH_SLOTS], + fresh_r: vec![false; FRESH_SLOTS], } ); } @@ -1412,10 +1866,23 @@ fn summarize_body_blockers_with( } "switch" => { // The arm table hangs off the descr rather than the code - // bytes, so name every instruction start as a successor. That - // is a superset of the arms, which keeps the answer sound. - for &target in &starts { - push!(points, work, target, state.clone()); + // bytes. With the table in hand the successors are its arms + // plus the fallthrough a key outside the table takes + // (`bhimpl_switch`); without it every instruction start is + // named, a superset that keeps the answer sound but lets the + // state at one arm's switch reach every other arm. + match descr_operand_index(code, &d).and_then(|index| switch_targets(index)) { + Some(targets) => { + for target in targets { + push!(points, work, target, state.clone()); + } + push!(points, work, d.next_pc, state); + } + None => { + for &target in &starts { + push!(points, work, target, state.clone()); + } + } } } _ => { @@ -4032,7 +4499,34 @@ pub(crate) fn try_walker_inline_builtin_call( builtin_inline_decline!("wrapper body has no ref register", fnaddr); return Ok(None); } - if let Some(decline) = descent_decline(jitcode.index()) { + // PyPy installs the builtin `__len__` implementations through + // `use_special_method_shortcut`, so an exact builtin receiver enters its + // layout body without visiting the generic lookup/error arms. Admit that + // execution path even when the body-wide safety scan sees a blocker on a + // different arm. Suppressing the row restores the conservative scan and + // is the A/B proof that the generated descent, rather than a hand emitter, + // supplies the trace. + let builtin_len_shortcut = if receiver.is_none() + && r_args.len() == 3 + && pyre_interpreter::builtins::is_builtin_len_function(callable) + { + let concrete_receiver = match arg_concretes.get(2) { + Some(ConcreteValue::Ref(obj)) => *obj, + _ => pyre_object::PY_NULL, + }; + spec_gate(SpecFold::BuiltinLenDescent, || { + Ok::, DispatchError>( + unsafe { exact_builtin_len_shortcut_receiver(concrete_receiver) }.then_some(()), + ) + })? + .is_some() + } else { + false + }; + let wrapper_item_count = usize::from(receiver.is_some()) + (r_args.len() - 2); + if !builtin_len_shortcut + && let Some(decline) = descent_decline(jitcode.index(), &[(0, wrapper_item_count)]) + { if matches!(decline, DescentDecline::Helper(_)) { log_descent_unlowered_helper_blockers(jitcode.index()); } @@ -4179,7 +4673,6 @@ pub(crate) fn try_walker_inline_builtin_call( // holds no `getarrayitem_gc_r` to name the item descriptor and the seeding // loop below has nothing to seed. Require the descriptor only when an // element is actually published. - let wrapper_item_count = usize::from(receiver.is_some()) + (r_args.len() - 2); let wrapper_args_descr_index = match wrapper_args_item_descr_index(body.code) { Some(index) => Some(index), None if wrapper_item_count == 0 => None, @@ -4409,11 +4902,16 @@ pub(crate) fn try_walker_inline_builtin_call( { if fbw_inline_diag_enabled() { eprintln!( - "[subwalk-abort] name={} pc={} abort_pc={} symbolic={:#x} disp=rollback", + "[subwalk-abort] name={} pc={} abort_pc={} symbolic={:#x} \ + disp=rollback effects={}/{} unjournaled={}/{}", unsafe { pyre_interpreter::function_get_name(callable) }, op.pc, pc, symbolic as u64, + effects_before, + fbw_executed_effect_count(), + unjournaled_before, + fbw_has_unjournaled_effect(), ); } cut_declined_subwalk(ctx, pre_fold_pos); @@ -4423,11 +4921,16 @@ pub(crate) fn try_walker_inline_builtin_call( if let DispatchError::OrthodoxSubWalkTraceUnsupported { pc, symbolic } = &error { if fbw_inline_diag_enabled() { eprintln!( - "[subwalk-abort] name={} pc={} abort_pc={} symbolic={:#x} disp=propagate", + "[subwalk-abort] name={} pc={} abort_pc={} symbolic={:#x} \ + disp=propagate effects={}/{} unjournaled={}/{}", unsafe { pyre_interpreter::function_get_name(callable) }, op.pc, pc, *symbolic as u64, + effects_before, + fbw_executed_effect_count(), + unjournaled_before, + fbw_has_unjournaled_effect(), ); } // The descent's `pc` is an offset into the callee's own @@ -8894,17 +9397,98 @@ pub(crate) fn try_walker_inline_property_get( dst: usize, dst_bank: char, ) -> Result, DispatchError> { - if !ctx.is_authoritative_executor || dst_bank != 'r' || ctx.fbw_mode.inline_subwalk { + let Some(name) = walker_load_name_from_code(w_code_ptr, name_idx) else { + return Ok(None); + }; + try_walker_inline_property_get_named( + ctx, + op, + code, + r_args, + call_descr, + obj, + Wtf8::new(name.as_str()), + dst, + dst_bank, + None, + ) +} + +/// Builtin `getattr(obj, name)` counterpart of +/// [`try_walker_inline_property_get`]. PyPy traces through the builtin and +/// the property's Python getter regardless of whether `name` has a UTF-8 +/// spelling; keep that route for pyre's opaque builtin-call boundary. +pub(crate) fn try_walker_inline_builtin_getattr_property( + ctx: &mut WalkContext<'_, '_, Sym>, + op: &DecodedOp, + code: &[u8], + r_args: &[OpRef], + call_descr: &dyn majit_ir::descr::CallDescr, + dst: usize, +) -> Result, DispatchError> { + if r_args.len() != 4 { return Ok(None); } - let Some(concrete_obj) = walker_concrete_ref_object(ctx, obj) else { + let concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(callable), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(concrete_obj), + ConcreteValue::Ref(concrete_name), + ) = (concretes[0], concretes[1], concretes[2], concretes[3]) + else { return Ok(None); }; - let Some(name) = walker_load_name_from_code(w_code_ptr, name_idx) else { + if callable.is_null() + || !null_or_self.is_null() + || concrete_obj.is_null() + || concrete_name.is_null() + || !pyre_interpreter::builtins::is_builtin_getattr_function(callable) + || !unsafe { pyre_object::is_exact_type(concrete_name, &pyre_object::STR_TYPE) } + { + return Ok(None); + } + let name = unsafe { pyre_object::w_str_get_wtf8(concrete_name) }; + try_walker_inline_property_get_named( + ctx, + op, + code, + r_args, + call_descr, + r_args[2], + name, + dst, + 'r', + Some((r_args[0], callable, r_args[3], concrete_name)), + ) +} + +#[allow(clippy::too_many_arguments)] +fn try_walker_inline_property_get_named( + ctx: &mut WalkContext<'_, '_, Sym>, + op: &DecodedOp, + code: &[u8], + r_args: &[OpRef], + call_descr: &dyn majit_ir::descr::CallDescr, + obj: OpRef, + name: &Wtf8, + dst: usize, + dst_bank: char, + builtin_guards: Option<( + OpRef, + pyre_object::PyObjectRef, + OpRef, + pyre_object::PyObjectRef, + )>, +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor || dst_bank != 'r' || ctx.fbw_mode.inline_subwalk { + return Ok(None); + } + let Some(concrete_obj) = walker_concrete_ref_object(ctx, obj) else { return Ok(None); }; let Some((w_type, version_tag, w_descr, fget)) = (unsafe { - pyre_interpreter::objspace::std::mapdict::property_get_fast_path(concrete_obj, &name) + pyre_interpreter::objspace::std::mapdict::property_get_fast_path_wtf8(concrete_obj, name) }) else { return Ok(None); }; @@ -8937,6 +9521,29 @@ pub(crate) fn try_walker_inline_property_get( // own past this point, so keep a rewind point the way the type-call fold // does. let pre_fold_pos = ctx.trace_ctx.get_trace_position(); + if let Some((callable_op, callable, name_op, concrete_name)) = builtin_guards { + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(callable as i64); + walker_emit_fold_guard_with_snapshot( + ctx, + op.pc, + OpCode::GuardValue, + &[callable_op, expected], + )?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + if !name_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(concrete_name as i64); + walker_emit_fold_guard_with_snapshot( + ctx, + op.pc, + OpCode::GuardValue, + &[name_op, expected], + )?; + } + } walker_pin_descriptor_slot(ctx, op.pc, w_descr, crate::descr::property_fget_descr())?; let inlined = try_walker_inline_resolved_user_call( ctx, @@ -11247,6 +11854,47 @@ pub(crate) fn finish_inline_callee_return( result } +#[allow(clippy::too_many_arguments)] +fn run_inline_call_subwalk( + ctx: &mut WalkContext<'_, '_, Sym>, + pc: usize, + descr_index: usize, + sub_body: &SubJitCodeBody, + int_args: &[OpRef], + int_arg_concretes: &[ConcreteValue], + ref_args: &[OpRef], + ref_arg_concretes: &[ConcreteValue], + float_args: &[OpRef], +) -> Result { + let uses_global_descr_pool = ctx + .raw_descrs + .runtime_jitcode_at(descr_index) + .is_some_and(|jitcode| jitcode.uses_global_descr_pool()); + if !uses_global_descr_pool { + return run_sub_jitcode_walk( + ctx, + pc, + sub_body, + int_args, + int_arg_concretes, + ref_args, + ref_arg_concretes, + float_args, + ); + } + + super::specialize::run_codewriter_helper_inline_call( + ctx, + pc, + sub_body, + int_args, + int_arg_concretes, + ref_args, + ref_arg_concretes, + float_args, + ) +} + thread_local! { /// Active explicit sub-walk driver for this OS thread. The pointer is /// scoped by `SubWalkDriverGuard` and is only dereferenced synchronously by @@ -11552,7 +12200,15 @@ impl<'a, Sym: WalkSym> SubWalkDriver<'a, Sym> { /// sub-walk driver is active. A nested CALL yields and replays that one step; /// these marks prove the replay crosses no concrete effect and delimit the IR /// preamble to cut. +/// +/// `pyjitpl.py MetaInterp.perform_call` changes frames only at a call. The +/// local replay continuation needs a heap-cache checkpoint for that CALL's +/// preamble, not for scalar operations, field reads or control flow. Both +/// direct inline calls and descents entered through residual-call handlers +/// can suspend. A non-call clears the checkpoint, so an unexpected suspension +/// fails at the driver's existing assertion rather than reusing stale state. pub(crate) fn note_subwalk_driver_step( + opname: &str, trace_position: majit_metainterp::recorder::TracePosition, heap_cache: &majit_metainterp::heapcache::HeapCache, ) { @@ -11570,10 +12226,59 @@ pub(crate) fn note_subwalk_driver_step( exchange.step_trace_position = Some(trace_position); exchange.step_effect_count = fbw_executed_effect_count(); exchange.step_unjournaled = fbw_has_unjournaled_effect(); - exchange.step_heap_cache = Some(heap_cache.clone()); + exchange.step_heap_cache = (opname.starts_with("inline_call_") + || opname.starts_with("residual_call_")) + .then(|| heap_cache.clone()); }); } +#[cfg(test)] +mod subwalk_checkpoint_tests { + use super::*; + + #[test] + fn only_calls_keep_a_heap_cache_replay_checkpoint() { + let mut exchange = SubWalkExchange:: { + pending: None, + completed: None, + active_frame_id: 0, + next_frame_id: 1, + step_trace_position: None, + step_effect_count: 0, + step_unjournaled: false, + step_heap_cache: None, + }; + let _driver = SubWalkDriverGuard::install(&mut exchange); + let position = majit_metainterp::recorder::TracePosition { + _pos: 0, + _count: 0, + _index: 0, + snapshot_data_len: 0, + snapshot_array_data_len: 0, + guard_count: Some(0), + }; + let heap_cache = majit_metainterp::heapcache::HeapCache::new(); + for (opname, keeps_checkpoint) in [ + ("inline_call_r_r", true), + ("int_add", false), + ("residual_call_ir_i", true), + ("getfield_gc_r", false), + ("goto_if_not", false), + ("live", false), + ("inline_call_irf_v", true), + ("int_return", false), + ] { + note_subwalk_driver_step::(opname, position, &heap_cache); + assert_eq!( + exchange.step_heap_cache.is_some(), + keeps_checkpoint, + "{opname}" + ); + assert_eq!(exchange.step_trace_position, Some(position)); + } + } +} + /// Seed a callee jitcode's register banks with positional args and walk /// its body, returning the callee's terminal [`DispatchOutcome`] /// (`SubReturn` / `SubRaise` / `Terminate` / `SwitchToBlackhole`). @@ -11829,8 +12534,17 @@ pub(crate) fn dispatch_inline_call_dr_kind( let (args, arg_width) = read_ref_var_list(code, op, 2, ctx)?; let arg_concretes = read_ref_var_list_concrete(code, op, 2, ctx); - let callee_result = - run_sub_jitcode_walk(ctx, op.pc, &sub_body, &[], &[], &args, &arg_concretes, &[]); + let callee_result = run_inline_call_subwalk( + ctx, + op.pc, + descr_index, + &sub_body, + &[], + &[], + &args, + &arg_concretes, + &[], + ); let callee_outcome = callee_result?; match callee_outcome { @@ -12034,9 +12748,10 @@ pub(crate) fn dispatch_inline_call_dir_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } - let callee_outcome = run_sub_jitcode_walk( + let callee_outcome = run_inline_call_subwalk( ctx, op.pc, + descr_index, &sub_body, &int_args, &int_arg_concretes, @@ -12236,9 +12951,10 @@ pub(crate) fn dispatch_inline_call_dirf_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } - let callee_result = run_sub_jitcode_walk( + let callee_result = run_inline_call_subwalk( ctx, op.pc, + descr_index, &sub_body, &int_args, &int_arg_concretes, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 2ff789e09ea..d6d622ae36b 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -403,6 +403,16 @@ impl<'a> RawDescrPool<'a> { Self::PerFn(descrs) => descrs.len(), } } + + fn runtime_jitcode_at( + self, + idx: usize, + ) -> Option> { + match self { + Self::Global => None, + Self::PerFn(descrs) => descrs.get(idx).and_then(|descr| descr.as_jitcode_owned()), + } + } } /// A callee local slot's recording-time concrete, tagged with the frame @@ -3531,6 +3541,11 @@ pub fn step( ctx: &mut WalkContext<'_, '_, Sym>, ) -> Result<(DispatchOutcome, usize), DispatchError> { let op: DecodedOp = decode_op_at(code, pc).ok_or(DispatchError::UndecodableOpcode { pc })?; + inline_call::note_subwalk_driver_step::( + op.opname, + ctx.trace_ctx.get_trace_position(), + ctx.trace_ctx.heap_cache(), + ); // The walker mixes translated vable operations (which update the shadow) // with concrete interpreter steps (which update the heap PyFrame). Pull // those concrete writes into `virtualizable_boxes` before any handler can @@ -3550,7 +3565,7 @@ pub fn step( // Consume it first so the marker on the following real op captures this // opcode's own live anchor rather than the preceding one's. if op.opname != "live" && !op.key.starts_with("jit_merge_point") { - if let Some(outcome) = record_python_debug_merge_point(ctx, op.pc)? { + if let Some(outcome) = record_python_debug_merge_point(ctx, code, op.pc)? { return Ok((outcome, op.next_pc)); } } @@ -3637,10 +3652,6 @@ pub fn walk( let callee = fbw_state::fbw_innermost_inline_callee_key(ctx); return Err(fbw_state::fbw_decline_inline_callee(ctx, pc, callee)); } - inline_call::note_subwalk_driver_step::( - ctx.trace_ctx.get_trace_position(), - ctx.trace_ctx.heap_cache(), - ); let (outcome, next_pc) = match step(code, pc, ctx) { Ok(stepped) => stepped, // Not an abort: a nested inline_call asked the heap-owned @@ -5385,32 +5396,15 @@ fn guard_current_frame_globals_identity( return Ok(false); } } - // `pyjitpl.py _establish_nullity`: a box's nullity is immutable, so a - // guard the trace already carries answers for every later fold site. - // Every LOAD_GLOBAL in the frame reaches here, and a trace that - // re-records the same guard per site spends its `trace_limit` on ops - // the optimizer then removes. - let known = ctx - .trace_ctx - .heap_cache() - .is_nullity_known(debugdata_op, |op| { - op.inline_const_to_value().and_then(|v| match v { - majit_ir::Value::Int(n) => Some(n), - majit_ir::Value::Ref(gc) => Some(gc.0 as i64), - _ => None, - }) - }); - if known != Some(present) { - let opcode = if present { - OpCode::GuardNonnull - } else { - OpCode::GuardIsnull - }; - walker_emit_fold_guard_with_snapshot(ctx, op_pc, opcode, &[debugdata_op])?; - ctx.trace_ctx - .heap_cache_mut() - .nullity_now_known(debugdata_op, present); - } + // Every LOAD_GLOBAL in the frame reaches here, and the emitter's + // `_establish_nullity` consult is what keeps the second one from + // spending `trace_limit` on a guard the first already carries. + let opcode = if present { + OpCode::GuardNonnull + } else { + OpCode::GuardIsnull + }; + walker_emit_fold_guard_with_snapshot(ctx, op_pc, opcode, &[debugdata_op])?; if !present { // No payload, so the namespace IS `pycode.w_globals` — the constant // already compared above. @@ -8149,6 +8143,7 @@ impl ActiveResumeFrame { /// transient, then resume without the caught exception. fn record_python_debug_merge_point( ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], jit_pc: usize, ) -> Result, DispatchError> { let Some(active) = ActiveResumeFrame::current(ctx.session, ctx.fbw_mode.snapshot_sym) else { @@ -8157,6 +8152,17 @@ fn record_python_debug_merge_point( if active.0.code_ptr.is_null() || !active.0.metadata.built_as_portal { return Ok(None); } + // `jit_pc` indexes the body being walked, and `py_floor_by_jit_pc` + // indexes the active frame's. A helper sub-walk pushes no Python frame, + // so the active frame stays the caller's while the offsets restart from + // zero inside the helper, and a small offset that happens to land on one + // of the caller's opcode boundaries reads back as a dispatch of the + // caller's first instructions. `debug_merge_point` belongs to + // `opimpl_jit_merge_point` (`pyjitpl.py`), which only the portal reaches, + // so require the walked body to be the frame's own. + if !std::ptr::eq(code.as_ptr(), active.0.jitcode.code.as_slice().as_ptr()) { + return Ok(None); + } let Ok(boundary_index) = active .0 .metadata @@ -9603,17 +9609,64 @@ fn walker_coerce_dispatching_operand_to_float( /// Emit a walker-native guard (`record_guard` + the walker snapshot for /// the just-recorded guard). Mirrors `MIFrame::generate_guard` for the /// full-body walk. +/// +/// A guard whose subject is a constant is not recorded, matching +/// `pyjitpl.py generate_guard`'s `if isinstance(box, Const): return`: the +/// property the guard would test is already decided by the constant, so the +/// op is dead weight the trace still pays for against `trace_limit`. The +/// first argument of a data guard is its subject; a control-flow guard +/// (`GUARD_NOT_INVALIDATED`, ...) passes none and is always recorded, which is +/// the `box=None` arm of the same test. fn walker_emit_guard_with_snapshot( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, opcode: OpCode, args: &[OpRef], ) -> Result<(), DispatchError> { + let subject = args.first().copied(); + if subject.is_some_and(|arg| arg.is_constant()) { + return Ok(()); + } + // `pyjitpl.py _establish_nullity`: a box's nullity is immutable, so the + // first guard on it answers for every later site, and `heapcache.py + // is_nullity_known` is what the second site asks. A fold reaches a + // receiver the surrounding opcode has already proved non-null often + // enough that the repeat is a whole operand's worth of `trace_limit`. + let nullity = match opcode { + OpCode::GuardNonnull => Some(true), + OpCode::GuardIsnull => Some(false), + _ => None, + }; + if let (Some(subject), Some(is_nonnull)) = (subject, nullity) + && ctx + .trace_ctx + .heap_cache() + .is_nullity_known(subject, walker_inline_const_word) + == Some(is_nonnull) + { + return Ok(()); + } stamp_guard_value_concrete(ctx.trace_ctx, opcode, args); ctx.trace_ctx.record_guard(opcode, args, 0); + if let (Some(subject), Some(is_nonnull)) = (subject, nullity) { + ctx.trace_ctx + .heap_cache_mut() + .nullity_now_known(subject, is_nonnull); + } walker_capture_snapshot_for_last_guard(ctx, op_pc) } +/// The word a constant `OpRef` stands for, in the shape +/// `heapcache::is_nullity_known` wants: RPython reads `box.getref_base()` off +/// the box itself, and the Rust constant namespace needs the pool consulted. +fn walker_inline_const_word(op: OpRef) -> Option { + op.inline_const_to_value().and_then(|v| match v { + majit_ir::Value::Int(n) => Some(n), + majit_ir::Value::Ref(gc) => Some(gc.0 as i64), + _ => None, + }) +} + /// A recording-path `GUARD_VALUE(box, const)` has already observed equality. /// Mirror `FrontendOp.value` by attaching that constant to the guarded box so /// bridge recipe construction sees the same fact as the runtime guard. @@ -9626,26 +9679,20 @@ fn stamp_guard_value_concrete(trace_ctx: &mut TraceCtx, opcode: OpCode, args: &[ } } -/// Fold-specific guard snapshot: records the guard and delegates to the -/// standard `walker_capture_snapshot_for_last_guard` which handles both the -/// FBW path (fresh `collect_outer_active_boxes` from `fbw_mode.snapshot_sym`) -/// and the per-opcode arm path (`ctx.outer_active_boxes`). +/// [`walker_emit_guard_with_snapshot`] named for a hand-fold's guards. /// -/// Previous attempt used `ctx.outer_active_boxes` directly, which is correct -/// for the per-opcode arm entry but empty (`Vec::new()`) in the main FBW -/// walk (`dispatch_via_miframe`). The FBW path in -/// `walker_capture_snapshot_for_last_guard_impl` re-derives `py_pc` from -/// `op_pc` and computes a fresh active-box set per guard, matching the -/// decoder's liveness query. +/// A fold reaches this from both the per-opcode arm entry, where the outer +/// active-box set is `ctx.outer_active_boxes`, and the main full-body walk, +/// where that set is empty and the snapshot helper re-derives `py_pc` from +/// `op_pc` and recomputes the active boxes per guard. Both are the shared +/// emitter's business, so the two spellings record the same thing. fn walker_emit_fold_guard_with_snapshot( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, opcode: OpCode, args: &[OpRef], ) -> Result<(), DispatchError> { - stamp_guard_value_concrete(ctx.trace_ctx, opcode, args); - ctx.trace_ctx.record_guard(opcode, args, 0); - walker_capture_snapshot_for_last_guard(ctx, op_pc) + walker_emit_guard_with_snapshot(ctx, op_pc, opcode, args) } fn walker_flush_guard_not_invalidated( @@ -9674,25 +9721,6 @@ fn walker_int_eq_const( r } -/// Record `uint_lt(raw, const k)` and stamp its already-known concrete truth. -/// Used to guard a machine shift count into `[0, k)`: the x86 SHL/SAR encoding -/// masks the count mod 64, so a reused trace whose shift count leaves the range -/// must bail to the generic (bignum-capable) leg rather than shift by `count & -/// 63`. `uint_lt` folds the negative case in (a negative count reads as a huge -/// unsigned value `>= k`). -fn walker_uint_lt_const( - ctx: &mut WalkContext<'_, '_, Sym>, - raw: OpRef, - k: i64, - concrete_truth: i64, -) -> OpRef { - let k_const = ctx.trace_ctx.const_int(k); - let r = ctx.trace_ctx.record_op(OpCode::UintLt, &[raw, k_const]); - ctx.trace_ctx - .set_opref_concrete(r, majit_ir::Value::Int(concrete_truth)); - r -} - /// Record `float_eq(raw, const k)` and stamp its already-known concrete /// truth. Used to build the float-div zero-divisor precondition guard /// walker-native (the JIT representation of `floatobject.py _floatdiv`'s @@ -11916,13 +11944,16 @@ fn establish_nullity( Ok(value) } -/// The unary member of the fused-goto family. `pyjitpl.py` -/// `opimpl_goto_if_not_int_is_zero` records the condition and hands it to -/// `opimpl_goto_if_not` with `replace=False`: +/// The unary members of the fused-goto family. `pyjitpl.py` +/// `opimpl_goto_if_not_int_is_zero` and `opimpl_goto_if_not_int_is_true` record +/// the condition and hand it to `opimpl_goto_if_not` with `replace=False`: /// /// condbox = self.execute(rop.INT_IS_ZERO, box) /// self.opimpl_goto_if_not(condbox, target, orgpc, replace=False) /// +/// `replace=False` is why the condition is recorded and then dropped: it names +/// no register, so there is nothing for a later op to read it back from. +/// /// Operand layout `iL`: 1B int reg + 2B label. fn fused_goto_if_not_int_unary( code: &[u8], @@ -12153,6 +12184,10 @@ fn handle( // over: the plain arm records no `int_is_true`, and its `replace` // rewrites the *value* box to a constant -- for the fused form that box // is `x`, not the boolean the branch tested. + // + // Both spellings reach the stream: `jtransform::optimize_goto_if_not` + // folds `int_is_true` and `int_is_zero` alike, and the assembler keys + // each on its own byte. "goto_if_not_int_is_true/iL" => { fused_goto_if_not_int_unary(code, op, ctx, OpCode::IntIsTrue) } @@ -12276,6 +12311,8 @@ fn handle( // exec-generated unary family. "ptr_nonzero/r>i" => ptr_nullity_record(code, op, ctx, true), "ptr_iszero/r>i" => ptr_nullity_record(code, op, ctx, false), + "guard_class/r>i" => guard_class_record(code, op, ctx, 'i'), + "guard_class/r>r" => guard_class_record(code, op, ctx, 'r'), "int_guard_value/i" => guard_value_record(code, op, ctx, GuardValueBank::Int), "ref_guard_value/r" => guard_value_record(code, op, ctx, GuardValueBank::Ref), "float_guard_value/f" => guard_value_record(code, op, ctx, GuardValueBank::Float), @@ -12785,9 +12822,6 @@ fn handle( w_class as pyre_object::PyObjectRef; } } - // `class_now_known` takes the vtable address: pyre tracks the - // concrete class pointer where upstream only raises HF_KNOWN_CLASS. - let known_class = descr.as_size_descr().map(|size| size.vtable() as i64); // pyjitpl.py `execute_new_with_vtable`. ctx.trace_ctx .profiler() @@ -12796,15 +12830,11 @@ fn handle( OpCode::NewWithVtable, majit_metainterp::counters::RECORDED_OPS, ); - let resbox = ctx - .trace_ctx - .record_op_with_descr(OpCode::NewWithVtable, &[], descr); - ctx.trace_ctx.heap_cache_mut().new_object(resbox); - if let Some(class) = known_class { + let resbox = ctx.trace_ctx - .heap_cache_mut() - .class_now_known(resbox, class); - } + .record_op_with_descr(OpCode::NewWithVtable, &[], descr.clone()); + ctx.trace_ctx.heap_cache_mut().new_object(resbox); + crate::helpers::note_class_word_after_new(ctx.trace_ctx, resbox, &descr); let dst = code[op.pc + 3] as usize; if let Some(value) = concrete { ctx.trace_ctx.set_opref_concrete(resbox, value); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 2daa18fb09f..e86fe226dcf 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -1856,6 +1856,38 @@ fn proxy_viewed_frame(obj: pyre_object::PyObjectRef) -> Option( + ctx: &mut WalkContext<'_, '_, Sym>, + allboxes: &[OpRef], +) -> bool { + let Some(&funcbox) = allboxes.first() else { + return false; + }; + let Some(majit_ir::Value::Int(addr)) = ctx.trace_ctx.box_value(funcbox) else { + // A funcbox the walk cannot read is one it cannot classify, and the + // executor declines on the same condition. + return true; + }; + majit_translate::codewriter::call::is_symbolic_fnaddr(addr) + || pyre_interpreter::is_abi_unsound_argument_residual(addr as usize) +} + /// Write the traced frame's locals region out before a residual that reads it /// through a live `FrameLocalsProxy`. /// @@ -1908,6 +1940,9 @@ fn write_back_locals_for_proxy_reader( ) else { return; }; + if residual_operands_are_not_all_objects(ctx, allboxes) { + return; + } let mut reads_traced_frame = false; for &arg in allboxes { let Some(obj) = walker_concrete_ref_object(ctx, arg) else { @@ -3395,6 +3430,27 @@ pub(crate) fn try_execute_residual_call_via_executor( continue; } if matches!(call_descr.arg_types().get(i), Some(majit_ir::Type::Ref)) && arg == 0 { + // The refusal names the helper and slot it fired on, because the + // repair for a helper that does check its NULL is a row in + // `mayforce_null_ref_arg_is_checked_sentinel` and the row needs + // both coordinates. A call the effect info gives no + // `RuntimeHelperKind` for has only the funcbox to name it by, so + // resolve that against the published registry too. + if fbw_debug_abort_enabled() { + let target = match allboxes.first().and_then(|&b| ctx.trace_ctx.box_value(b)) { + Some(majit_ir::Value::Int(addr)) => addr, + _ => 0, + }; + let name = pyre_interpreter::jit_trace_fnaddrs() + .into_iter() + .find(|&(_, addr)| addr == target) + .map_or("-", |(name, _)| name); + eprintln!( + "[nullref-refusal] helper={helper:?} target={name}/{target:#x} \ + arg_index={i} nargs={} pc={op_pc} opcode={call_opcode:?}", + args.len() + ); + } return Ok(declined_symbolic(call_opcode)); } } @@ -5284,42 +5340,33 @@ pub(crate) enum SpecializedBinop { /// Non-numeric operands stay an impure residual, so admitting them here would /// trigger the nested-residual 6421 abort storm. /// -/// The accepted set is every tag a specialization table lowers with no runtime -/// decline path left, so nothing survives as a residual. Both tables key the -/// in-place tag to the SAME arm as its plain form, so the two forms are -/// admitted together: +/// The accepted set is every tag handled either by the generated exact-int +/// descent or by the remaining float specialization without an unsafe replay +/// residual. In-place tags select the same concrete builtin arithmetic as +/// their plain forms, so the two forms are admitted together: /// -/// - `Add` / `Subtract` / `Multiply` (+ in-place) — `IntAddOvf` / `IntSubOvf` / -/// `IntMulOvf` in `try_walker_specialize_binary_op_int`, `FloatAdd` / -/// `FloatSub` / `FloatMul` in `try_walker_specialize_binary_op_float`. In -/// both tables `needs_concrete_check` is false, so either argument width -/// lowers unconditionally. +/// - `Add` / `Subtract` / `Multiply` (+ in-place) — the generated +/// `binary_value_from_tag` descent for exact ints, and `FloatAdd` / +/// `FloatSub` / `FloatMul` in `try_walker_specialize_binary_op_float`. /// - `And` / `Or` / `Xor` (+ in-place) — `IntAnd` / `IntOr` / `IntXor`, also /// unconditional, but *int-only*: the float table falls through to /// `_ => return Ok(None)` for them. Hence the separate /// [`SpecializedBinop::PlainInt`] arm, which additionally demands both /// operands be proven plain ints. -/// - `FloorDivide` / `Remainder` (+ in-place) — `IntFloorDiv` / `IntMod`, -/// int-only for the same reason (neither has a `FLOAT_*` opcode). These two -/// are the one accepted pair whose lowering *can* decline — on a zero divisor -/// or on `i64::MIN` by `-1` — but a surviving residual is still replay-safe -/// on its own merits: `int.__floordiv__` / `int.__mod__` read two immutable -/// boxes and either allocate a fresh result or raise `ZeroDivisionError`, -/// which commits nothing a replay would double. The `plain_int` proof is -/// what rules out a user `__mod__`. `i % k` in an `if` is the common shape -/// that would otherwise residualize the whole callee -/// (`bench/synth/gc_bug_bridge_flavor_traceback_names`). +/// - `FloorDivide` / `Remainder` (+ in-place) — exact-int-only generated +/// descent. Its success arm records the `int.py_div` / `int.py_mod` +/// oopspec, and its zero-divisor arm propagates the materialised +/// `W_BaseException` as `SubRaise`. The `plain_int` proof rules out a user +/// `__mod__`. /// /// Every other tag is excluded because its lowering can still decline and /// leave a residual that is NOT replay-safe on its own: /// /// - `TrueDivide` (+ in-place) — float-table only, and it declines a zero /// divisor so the raising `descr_truediv` stays recorded. -/// - `Lshift` (+ in-place) — the int table declines it outright (the reused -/// trace would bake a count the x86 `SHL` masks mod 64, and the guarded form -/// breaks the cranelift bridge). -/// - `Rshift` (+ in-place) — declines a negative or `>= LONG_BIT` count rather -/// than baking intobject.py's fold-to-`0`/`-1`. +/// - shifts (+ in-place) — the generated descent handles exact-int sites, but +/// this replay admission remains conservative around the count-dependent +/// exception and large-count arms. /// - `Power` (+ in-place) — the int table has no arm; the float table inlines /// `_pow` but keeps a cold-path residual for nan/inf/negative-base operands. /// - `Subscr`, `MatrixMultiply` (+ in-place) — no arm in either table. @@ -6631,25 +6678,6 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } - // `len(x)` on an exact canonical list: inline the strategy-guarded - // length read (guard_value callable + guard_class + exact w_class + - // guard_value strategy + length getfield + wrapint) instead of the - // opaque `bh_call_fn(len_builtin, NULL, x)` residual — the shape the - // meta-tracer produces upstream (descroperation.py `_len` → - // `W_ListObject.length()`). Read-only like the SUBSCR fold, so no - // sub-walk restriction; any non-matching shape falls through to the - // generic residual (SAFE). - if ctx.is_authoritative_executor - && dst_bank == 'r' - && foldable_runtime_helper == majit_ir::RuntimeHelperKind::CallFn - && spec_gate(SpecFold::BuiltinLen, || { - try_walker_specialize_builtin_len(ctx, code, op, &r_args, dst) - })? - .is_some() - { - return Ok((DispatchOutcome::Continue, op.next_pc)); - } - // `isinstance(x, C)` for a class whose metaclass is exactly `type`: the // answer is `issubtype`'s elidable MRO test on two promoted types // (typeobject.py), so pin both and bake it instead of leaving the opaque @@ -6974,93 +7002,6 @@ pub(crate) fn dispatch_residual_call_iRd_kind( } } - // UNARY_POSITIVE. Descend `pos_inner` -- `pos` past the override probe -- - // rather than re-emit its identity arm by hand, the same shape the invert - // and neg descents take. Sits ahead of the `unary_positive_int` fold so - // that fold's `consulted` count reads whether the descent took the site. - if ctx.is_authoritative_executor - && dst_bank == 'r' - && r_args.len() == 1 - && foldable_runtime_helper == majit_ir::RuntimeHelperKind::UnaryPositive - && spec_gate(SpecFold::UnaryPositiveDescent, || { - try_walker_orthodox_unary_positive(ctx, op.pc, r_args[0], dst, dst_bank) - })? - .is_some() - { - return Ok((DispatchOutcome::Continue, op.next_pc)); - } - - // #61: UNARY_POSITIVE `+int` identity fold. Kept behind the descent so a - // build whose `pos_inner` jitcode is missing or unlowered still forwards - // an exact-int operand. A bool (`+True` is int `1`) / non-int operand - // declines to the generic leg so its `__pos__` still runs. - if ctx.is_authoritative_executor - && dst_bank == 'r' - && r_args.len() == 1 - && foldable_runtime_helper == majit_ir::RuntimeHelperKind::UnaryPositive - && spec_gate(SpecFold::UnaryPositiveInt, || { - try_walker_specialize_unary_positive_int(ctx, op.pc, r_args[0], dst, dst_bank) - })? - .is_some() - { - return Ok((DispatchOutcome::Continue, op.next_pc)); - } - - // UNARY_NEGATIVE. Descend `neg_inner` -- `neg` past the override probe -- - // rather than re-emit its integer arm by hand, the same shape the invert - // descent below takes. The translated body owns the ordinary int arm and - // the exact `long` operand. Bool, subclass and non-int operands still fall - // through to the generic residual so their override semantics are - // preserved, and the `INT_MIN` promotion declines to the fold below. - if ctx.is_authoritative_executor - && dst_bank == 'r' - && r_args.len() == 1 - && foldable_runtime_helper == majit_ir::RuntimeHelperKind::UnaryNegative - && spec_gate(SpecFold::UnaryNegativeDescent, || { - try_walker_orthodox_unary_negative(ctx, op.pc, r_args[0], dst, dst_bank) - })? - .is_some() - { - return Ok((DispatchOutcome::Continue, op.next_pc)); - } - - // #61: UNARY_NEGATIVE `-int`. Kept behind the descent, which declines the - // one operand `descr_neg` promotes: the fold pins that operand with - // `guard_value` and takes the `_make_ovf2long` tail, so the `2**63` long is - // a constant the ops reading it fold against. It also serves an exact-int - // operand in a build whose `neg_inner` jitcode is missing or unlowered. - if ctx.is_authoritative_executor - && dst_bank == 'r' - && r_args.len() == 1 - && foldable_runtime_helper == majit_ir::RuntimeHelperKind::UnaryNegative - && spec_gate(SpecFold::UnaryNegativeInt, || { - try_walker_specialize_unary_negative_int( - ctx, op.pc, r_args[0], &allboxes, call_descr, dst, dst_bank, - ) - })? - .is_some() - { - return Ok((DispatchOutcome::Continue, op.next_pc)); - } - - // UNARY_INVERT. Descend `invert_inner` -- `invert` past the override probe - // and the bool slot -- rather than re-emit its integer arm by hand. This - // replaced `unary_invert_int`, whose site it took whole: with the descent - // in, that fold measured `consulted=0` on every fixture that exercises `~`. - // Falls through to the generic residual when the body is absent from this - // build or reaches a helper the build did not lower. - if ctx.is_authoritative_executor - && dst_bank == 'r' - && r_args.len() == 1 - && foldable_runtime_helper == majit_ir::RuntimeHelperKind::UnaryInvert - && spec_gate(SpecFold::UnaryInvertDescent, || { - try_walker_orthodox_unary_invert(ctx, op.pc, r_args[0], dst, dst_bank) - })? - .is_some() - { - return Ok((DispatchOutcome::Continue, op.next_pc)); - } - // #62: specialize STORE_SUBSCR `list[int] = value` (int / float storage, // in-bounds, type-matching) to the walker-native `setarrayitem_raw` form, // eliding the `CALL_MAY_FORCE` that would force the virtualizable every @@ -7306,6 +7247,19 @@ pub(crate) fn dispatch_residual_call_iRd_kind( return Ok((DispatchOutcome::Continue, op.next_pc)); } + // PyPy traces through builtin `getattr` into a property's Python getter. + // The ordinary LOAD_ATTR spelling has the same inline route above; keep + // the builtin spelling on it too, including lone-surrogate names carried + // by PyPy's RPython-string representation. + if ctx.is_authoritative_executor + && dst_bank == 'r' + && foldable_runtime_helper == majit_ir::RuntimeHelperKind::CallFn + && let Some(inlined) = + try_walker_inline_builtin_getattr_property(ctx, op, code, &r_args, call_descr, dst)? + { + return Ok(inlined); + } + // `getattr(type, name)` whose class-MRO value is returned unchanged: // pin receiver, name, and the receiver version, then use the green value. // Non-matching shapes fall through to the generic residual. @@ -7885,7 +7839,7 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // pyjitpl.py `execute_and_record_varargs`; may-force // calls use `history.record_nospec` and therefore count nothing. - if matches!( + let profiled_call = matches!( call_opcode, OpCode::CallI | OpCode::CallR @@ -7899,13 +7853,11 @@ pub(crate) fn dispatch_residual_call_iRd_kind( | OpCode::CallLoopinvariantR | OpCode::CallLoopinvariantF | OpCode::CallLoopinvariantN - ) { + ); + if profiled_call { ctx.trace_ctx .profiler() .count_ops(call_opcode, majit_metainterp::counters::OPS); - ctx.trace_ctx - .profiler() - .count_ops(call_opcode, majit_metainterp::counters::RECORDED_OPS); } // Always record `list_write_barrier` on the Object strategy's in-place // append arm. Dropping it in favour of the backend's @@ -7922,6 +7874,13 @@ pub(crate) fn dispatch_residual_call_iRd_kind( let recorded = ctx .trace_ctx .record_op_with_descr(call_opcode, &allboxes, descr.clone()); + // `_record_helper_varargs` counts RECORDED_OPS as it records; a + // may-force call takes `history.record_nospec` and counts nothing. + if profiled_call { + ctx.trace_ctx + .profiler() + .count_ops(call_opcode, majit_metainterp::counters::RECORDED_OPS); + } // `MIFrame.execute_varargs(pure=True)` parity: for // `CallPure*` whose every argbox carries a known `box_value`, @@ -9096,6 +9055,12 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // (BoxInt exec, generic residual below) requires every box bound. ensure_residual_call_args_bound(&allboxes, op.pc)?; + // `w_bool_from(truth)` inside a descended body: guard the truth and take + // the singleton, as `space.newbool` traces. + if try_walker_fold_newbool_call(ctx, op.pc, &allboxes, &i_args, dst, dst_bank)?.is_some() { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + // BoxInt fold (#62): `box_int_fn(raw)` allocates a fresh `PyLong`. The // opaque CanRaise residual the generic leg would record blocks the // optimizer (no DCE of an unused/round-tripped box). Emit the @@ -9158,15 +9123,33 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( ) })? } else { - // int specialization first; float (incl. mixed int/float) - // as a fallback so two-int operands keep int arithmetic. - if let Some(outcome) = spec_gate(SpecFold::BinaryOpInt, || { - try_walker_specialize_binary_op_int( - ctx, op.pc, op_tag, &r_args, &allboxes, call_descr, dst, dst_bank, + // The exact-int zero-divisor raise runs BEFORE the + // descent: the descended body reaches its + // `ZeroDivisionError` through an opaque published + // materialiser, whose concrete result no longer + // virtualizes (see + // `try_walker_specialize_binary_op_int_zero_div`). + if let Some(outcome) = spec_gate(SpecFold::BinaryOpIntZeroDiv, || { + try_walker_specialize_binary_op_int_zero_div( + ctx, op.pc, op_tag, &r_args, &allboxes, call_descr, dst_bank, + ) + })? { + return Ok((outcome, op.next_pc)); + } + // Descend the helper whole ahead of the hand folds, so + // their `consulted` counts read whether the descent + // took the site. + if let Some(outcome) = spec_gate(SpecFold::BinaryOpDescent, || { + try_walker_orthodox_binary_op( + ctx, op.pc, op_tag, tag_opref, &r_args, dst, dst_bank, ) })? { return Ok((outcome, op.next_pc)); } + // Float (including mixed int/float) remains a hand + // fallback. Exact int pairs have already been taken + // whole by `binary_op_descent`, including overflow and + // zero-division exception arms. // longobject.py `_make_generic_descr_binop` and // `descr_sub` use the rbigint.int_* family for // mixed Long/Int operands. @@ -9293,6 +9276,16 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // operand layout whose class compares `is_w` by value. try_walker_fold_is_op(ctx, op.pc, op_tag, &r_args, dst, dst_bank)? } else { + // Descend the helper whole ahead of the hand folds, so + // their `consulted` counts read whether the descent + // took the site. + if let Some(outcome) = spec_gate(SpecFold::CompareOpDescent, || { + try_walker_orthodox_compare_op( + ctx, op.pc, op_tag, tag_opref, &r_args, dst, dst_bank, + ) + })? { + return Ok((outcome, op.next_pc)); + } // int compare first; then long (two-bigint operands keep // bigint comparison); float (incl. mixed int/float) last. match spec_gate(SpecFold::CompareOpInt, || { @@ -9447,7 +9440,7 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( write_back_locals_for_proxy_reader(ctx, &allboxes); } - if matches!( + let profiled_call = matches!( call_opcode, OpCode::CallI | OpCode::CallR @@ -9461,13 +9454,11 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( | OpCode::CallLoopinvariantR | OpCode::CallLoopinvariantF | OpCode::CallLoopinvariantN - ) { + ); + if profiled_call { ctx.trace_ctx .profiler() .count_ops(call_opcode, majit_metainterp::counters::OPS); - ctx.trace_ctx - .profiler() - .count_ops(call_opcode, majit_metainterp::counters::RECORDED_OPS); } // `pyjitpl.py:1943` takes `patch_pos` before recording the call so // `record_result_of_call_pure` can cut it back out. @@ -9475,6 +9466,13 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( let recorded = ctx .trace_ctx .record_op_with_descr(call_opcode, &allboxes, descr.clone()); + // `_record_helper_varargs` counts RECORDED_OPS as it records; a + // may-force call takes `history.record_nospec` and counts nothing. + if profiled_call { + ctx.trace_ctx + .profiler() + .count_ops(call_opcode, majit_metainterp::counters::RECORDED_OPS); + } // `MIFrame.execute_varargs(pure=True)` parity — see // `dispatch_residual_call_iRd_kind` for the upstream walk. @@ -9729,7 +9727,7 @@ pub(crate) fn dispatch_residual_call_iIRFd_kind( write_back_locals_for_proxy_reader(ctx, &allboxes); } - if matches!( + let profiled_call = matches!( call_opcode, OpCode::CallI | OpCode::CallR @@ -9743,13 +9741,11 @@ pub(crate) fn dispatch_residual_call_iIRFd_kind( | OpCode::CallLoopinvariantR | OpCode::CallLoopinvariantF | OpCode::CallLoopinvariantN - ) { + ); + if profiled_call { ctx.trace_ctx .profiler() .count_ops(call_opcode, majit_metainterp::counters::OPS); - ctx.trace_ctx - .profiler() - .count_ops(call_opcode, majit_metainterp::counters::RECORDED_OPS); } // `pyjitpl.py:1943` takes `patch_pos` before recording the call so // `record_result_of_call_pure` can cut it back out. @@ -9757,6 +9753,13 @@ pub(crate) fn dispatch_residual_call_iIRFd_kind( let recorded = ctx .trace_ctx .record_op_with_descr(call_opcode, &allboxes, descr.clone()); + // `_record_helper_varargs` counts RECORDED_OPS as it records; a + // may-force call takes `history.record_nospec` and counts nothing. + if profiled_call { + ctx.trace_ctx + .profiler() + .count_ops(call_opcode, majit_metainterp::counters::RECORDED_OPS); + } // `MIFrame.execute_varargs(pure=True)` parity — see // `dispatch_residual_call_iRd_kind` for the upstream walk. diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index de466df823a..2340c74ac6d 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -697,6 +697,7 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // guard resume at the loop header instead of the guard's own // opcode, re-running loop iterations and corrupting the result. // Mirror of `MIFrame::publish_last_instr_to_vable`. + let mut async_force_vable_boxes = None; if sym.owns_virtualizable_shadow() { let last_instr_value = py_pc as i64 - 1; let last_instr_op = ctx.trace_ctx.const_int(last_instr_value); @@ -706,6 +707,25 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( last_instr_op, Value::Int(last_instr_value), ); + // `ResumeGuardForcedDescr.handle_async_forcing` consumes the + // virtualizable section WHILE the residual is running. Its + // result belongs to the MIFrame's post-call registers, but + // the interpreter has not pushed it yet. Preserve the stack + // before the post-call depth/operand overlay below, otherwise + // forcing reads an unwritten result home as a GC Ref. + // Do publish last_instr first: the walker does not maintain + // that scalar at each opcode, and callers inspecting this + // frame must see the current call, not the last merge point. + let guard_opcode = match scope.guard_stamp { + GuardStampTarget::LastOp => ctx.trace_ctx.last_op_opcode(), + GuardStampTarget::GuardFromEnd(from_end) => { + ctx.trace_ctx.guard_op_opcode_from_end(from_end) + } + }; + if matches!(guard_opcode, Some(OpCode::GuardNotForced)) { + async_force_vable_boxes = + Some(ctx.trace_ctx.build_snapshot_vable_vref_boxes().0); + } // Publish `valuestackdepth` for THIS guard's resume // coordinate the same way `last_instr` is published above. // `sym.valuestackdepth` is NOT usable: the walker never @@ -1192,6 +1212,7 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( ctx.trace_ctx.set_virtualizable_box_at(idx, value); } let (vable_boxes, vref_boxes) = ctx.trace_ctx.build_snapshot_vable_vref_boxes(); + let vable_boxes = async_force_vable_boxes.unwrap_or(vable_boxes); for (idx, old) in saved_shadow { if let Some(old) = old { ctx.trace_ctx.set_virtualizable_box_at(idx, old); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 118ef4fd3d6..14787abb698 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -186,9 +186,8 @@ pub(crate) fn try_walker_specialize_truth_int( // `__bool__`, and `walker_numeric_builtin_class` answers with the // canonical `int` — a `w_class` the recorded operand does not carry, so // the pin below becomes a guard that fails on the very value that - // recorded it. Decline before unboxing, as `walker_unary_int_operand` - // does; `walker_numeric_builtin_class` documents this gate as its - // precondition. + // recorded it. Decline before unboxing; `walker_numeric_builtin_class` + // documents this gate as its precondition. if !pyre_object::is_int(obj) || pyre_object::is_bool(obj) || !pyre_object::is_exact_builtin_instance(obj) @@ -240,74 +239,6 @@ pub(crate) fn try_walker_specialize_truth_bool( Ok(Some(truth)) } -/// #61: walker-native identity fold for the `UNARY_POSITIVE` residual -/// (oopspec [`majit_ir::RuntimeHelperKind::UnaryPositive`]). The object-space -/// `pos` on an exact int returns the operand unchanged, so a concrete non-bool -/// `W_IntObject` operand folds to the operand box itself behind the same guard -/// prefix the truth / binary int folds emit (a low-bit tag test for a tagged -/// immediate, `GUARD_CLASS INT` for a heap box). The unboxed raw is discarded -/// (DCE): the result is the box, not its `intval`. -/// -/// Returns `Ok(Some(()))` when the fold was emitted (caller returns -/// `Continue`); `Ok(None)` for a bool (`+True` is int `1`, not identity), a -/// numeric subclass, or a non-int operand — the caller then falls through to -/// the generic `CallMayForce` residual so a subclass / user `__pos__` still -/// runs. -pub(crate) fn try_walker_specialize_unary_positive_int( - ctx: &mut WalkContext<'_, '_, Sym>, - op_pc: usize, - operand: OpRef, - dst: usize, - dst_bank: char, -) -> Result, DispatchError> { - // `+x` is identity only for an EXACT builtin int. A bool shares the - // `intval` but `+True` is int `1`, not identity, and a numeric subclass - // must reach its own `__pos__` rather than have the operand forwarded. - let Some((_, x_class)) = walker_unary_int_operand(ctx, operand) else { - return Ok(None); - }; - let int_type_addr = &pyre_object::pyobject::INT_TYPE as *const _ as i64; - // Emit the guard prefix (`GUARD_CLASS INT` / tag test) so a later non-int - // arrival deopts; the returned raw is unused because the result is the - // operand box itself. - let _ = walker_unbox_int(ctx, op_pc, operand, int_type_addr)?; - walker_guard_exact_w_class(ctx, op_pc, operand, x_class)?; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, operand)?; - Ok(Some(())) -} - -/// Shared gate for the `UNARY_POSITIVE` / `UNARY_NEGATIVE` / `UNARY_INVERT` int -/// folds: the operand must be a concrete EXACT builtin non-bool `W_IntObject`. -/// A bool unboxes through its own `&BOOL_TYPE` guard (declined here for -/// simplicity — `+True` / `-True` / `~True` stay on the residual). -/// -/// Returns the concrete `intval` and the canonical `int` type object the caller -/// must pin with [`walker_guard_exact_w_class`]. `is_exact_builtin_instance` -/// only settles the operand the trace RECORDED; a numeric subclass keeps the -/// builtin `ob_type`, so the `GUARD_CLASS INT` the fold emits does not stop one -/// from entering the trace later and being answered by the fold instead of its -/// own `__neg__` / `__invert__` / `__pos__`. Pinning `w_class` is what makes -/// that arrival side-exit, and the operand that carries the null spelling of -/// "exact builtin" has no value to pin, so it declines — the same shape the -/// long folds use (`walker_exact_builtin_class` + guard). -fn walker_unary_int_operand( - ctx: &mut WalkContext<'_, '_, Sym>, - operand: OpRef, -) -> Option<(i64, pyre_object::PyObjectRef)> { - let obj = walker_concrete_ref_object(ctx, operand)?; - // SAFETY: `obj` is a live concrete `PyObjectRef` from the walker shadow. - unsafe { - if !pyre_object::is_int(obj) - || pyre_object::is_bool(obj) - || !pyre_object::is_exact_builtin_instance(obj) - { - return None; - } - let class = walker_exact_builtin_class(obj)?; - Some((pyre_object::w_int_get_value(obj), class)) - } -} - /// The `W_LongObject.value` payload of a concrete long, read the way the folds /// that pass a payload to an `rbigint` helper need it. /// @@ -338,150 +269,6 @@ fn walker_read_long_payload( payload } -/// `intobject.py _make_ovf2long`: the tail every int arithmetic fold shares -/// once its own guard has pinned the promoting branch — `GUARD_OVERFLOW` for -/// the `BINARY_OP` arm, `GUARD_VALUE` on the operand for unary negate. The tail -/// is the elidable raw-int bigint helper (`rbigint.py:717/788/873`) under -/// `EF_ELIDABLE_OR_MEMORYERROR`, then the inline `W_LongObject` box around the -/// payload it returns. `payload_fn` takes the two machine ints in -/// `(raw, concrete)` pairs, which is also the shape the concrete-args vector -/// wants. -/// -/// The box needs no preceding fits_int guard. `newlong_from_rbigint` -/// (objspace.py:316-320) demotes through `rbigint.toint()`, whose -/// `numdigits() > MAX_DIGITS_THAT_CAN_FIT_IN_INT` test (rbigint.py) that -/// guard already answers: the helper is the *exact* int-pair sum / difference / -/// product, so a value that just overflowed a machine int cannot fit one back. -/// The same fold is what lets `try_walker_specialize_binary_op_long_int_pow` -/// skip its result-fits guard. -fn walker_emit_ovf2long_box( - ctx: &mut WalkContext<'_, '_, Sym>, - op_pc: usize, - payload_fn: *const (), - lhs: (OpRef, i64), - rhs: (OpRef, i64), - boxed_result_i64: i64, -) -> Result { - let (lhs_raw, la) = lhs; - let (rhs_raw, rb) = rhs; - let payload_concrete = - unsafe { long_payload_of(boxed_result_i64 as usize as pyre_object::PyObjectRef) }; - let concrete_args = [ - majit_ir::Value::Int(payload_fn as usize as i64), - majit_ir::Value::Int(la), - majit_ir::Value::Int(rb), - ]; - let payload = ctx.trace_ctx.call_typed_with_effect_pure_can_raise( - OpCode::CallR, - payload_fn, - &[lhs_raw, rhs_raw], - &[majit_ir::Type::Int, majit_ir::Type::Int], - majit_ir::Type::Ref, - majit_metainterp::ELIDABLE_OR_MEMERROR_EFFECT_INFO, - &concrete_args, - majit_ir::Value::Ref(majit_ir::GcRef(payload_concrete as usize)), - ); - ctx.trace_ctx.set_opref_concrete( - payload, - majit_ir::Value::Ref(majit_ir::GcRef(payload_concrete as usize)), - ); - if payload.inline_const_to_value().is_none() { - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoException, &[])?; - } - let result = crate::helpers::emit_box_long_inline( - ctx.trace_ctx, - payload, - crate::descr::w_long_size_descr(), - crate::descr::long_value_descr(), - ); - ctx.trace_ctx.set_opref_concrete( - result, - majit_ir::Value::Ref(majit_ir::GcRef(boxed_result_i64 as usize)), - ); - Ok(result) -} - -/// #61: walker-native int specialization for the `UNARY_NEGATIVE` residual -/// (oopspec [`majit_ir::RuntimeHelperKind::UnaryNegative`]). `-x` on an exact -/// int is `0 - x`; the object-space `neg` promotes only `-INT_MIN` to a -/// `W_LongObject` (`intobject.py` `descr_neg` → `_make_ovf2long`). Since -/// majit has no overflow-checked unary negate, the fold expresses `-x` as -/// `IntSubOvf(0, x)` behind a `GUARD_CLASS INT`, reusing the binary-sub -/// overflow discipline in both directions: a record value other than `INT_MIN` -/// emits `GUARD_NO_OVERFLOW` so an `INT_MIN` arrival on the reused trace deopts -/// rather than wrapping back to `INT_MIN`, and a record value of `INT_MIN` -/// pins the operand with `GUARD_VALUE` and takes the same `_make_ovf2long` tail -/// the `BINARY_OP` overflow arm takes, so the `2**63` long is built from the -/// elidable bigint helper instead of the `CallMayForce` residual. -/// -/// Returns `Ok(Some(()))` when the fold was emitted (caller returns -/// `Continue`); `Ok(None)` for a bool / subclass / non-int operand, when the -/// residual result box is unavailable, or when an `INT_MIN` operand did not -/// produce the promoted `W_LongObject` the payload read expects. -pub(crate) fn try_walker_specialize_unary_negative_int( - ctx: &mut WalkContext<'_, '_, Sym>, - op_pc: usize, - operand: OpRef, - allboxes: &[OpRef], - call_descr: &dyn majit_ir::descr::CallDescr, - dst: usize, - dst_bank: char, -) -> Result, DispatchError> { - let Some((x, x_class)) = walker_unary_int_operand(ctx, operand) else { - return Ok(None); - }; - let Some(boxed_result_i64) = walker_execute_may_force_boxed(ctx, allboxes, call_descr) else { - return Ok(None); - }; - // `0 - INT_MIN` is the one operand `descr_neg` promotes. - let overflows = x == i64::MIN; - if overflows { - let boxed_result_obj = boxed_result_i64 as usize as pyre_object::PyObjectRef; - if boxed_result_obj == pyre_object::PY_NULL - || !unsafe { pyre_object::is_long(boxed_result_obj) } - { - return Ok(None); - } - } - let int_type_addr = &pyre_object::pyobject::INT_TYPE as *const _ as i64; - let x_raw = walker_unbox_int(ctx, op_pc, operand, int_type_addr)?; - walker_guard_exact_w_class(ctx, op_pc, operand, x_class)?; - let zero_raw = ctx.trace_ctx.const_int(0); - let boxed = if overflows { - // `0 - x` overflows an i64 for exactly one operand, so "the negate - // promoted" and "the operand is INT_MIN" name the same set: guarding - // the value admits what `GUARD_OVERFLOW` would and nothing more. The - // value form is the one the tail can use — with `x_raw` constant the - // elidable bigint call folds to the `2**63` payload it returned while - // recording instead of running once per iteration. This is the - // `guard_value` spelling the version-tag promotes already use. - let int_min = ctx.trace_ctx.const_int(i64::MIN); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardValue, &[x_raw, int_min])?; - walker_emit_ovf2long_box( - ctx, - op_pc, - pyre_object::longobject::jit_bigint_sub_int_int as *const (), - (zero_raw, 0), - (x_raw, x), - boxed_result_i64, - )? - } else { - let result_value = 0i64.wrapping_sub(x); - let raw_result = ctx - .trace_ctx - .record_op(OpCode::IntSubOvf, &[zero_raw, x_raw]); - ctx.trace_ctx - .set_opref_concrete(raw_result, majit_ir::Value::Int(result_value)); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoOverflow, &[])?; - let boxed = walker_box_int(ctx, op_pc, raw_result, result_value)?; - ctx.trace_ctx - .set_opref_concrete(boxed, box_int_concrete(result_value, boxed_result_i64)); - boxed - }; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, boxed)?; - Ok(Some(())) -} - /// #57: walker-native speculative int specialization for the `BINARY_OP` /// helper residual_call (oopspec `BinaryOp`). Re-derives /// the former int fast path's structure (`guard_class` + `getfield_gc_i` per @@ -501,355 +288,6 @@ pub(crate) fn try_walker_specialize_unary_negative_int( /// not both concrete `W_IntObject`, or an unsupported helper arm is reached — the caller /// then falls through to the generic `CallMayForce` record so the /// Python-level `__op__` semantics are preserved. -/// The raw machine int of an int/bool operand, for the specialized IR: guard -/// the operand's exact class, then load `intval` out of the box. -/// -/// A bool `space.newbool` produced this same walk arrives as the prebuilt -/// `w_True` / `w_False` singleton behind its own truth guard, so both the class -/// guard and the `intval` load read a constant and fold away — no side table -/// has to reconnect the box to the truth it was built from. -fn walker_int_operand_raw( - ctx: &mut WalkContext<'_, '_, Sym>, - op_pc: usize, - operand: OpRef, - operand_obj: pyre_object::PyObjectRef, - type_addr: i64, - intval_descr: majit_ir::DescrRef, -) -> Result { - let raw = walker_unbox_int_typed(ctx, op_pc, operand, type_addr, intval_descr)?; - walker_guard_exact_w_class( - ctx, - op_pc, - operand, - walker_numeric_builtin_class(operand_obj), - )?; - Ok(raw) -} - -pub(crate) fn try_walker_specialize_binary_op_int( - ctx: &mut WalkContext<'_, '_, Sym>, - op_pc: usize, - op_tag: i64, - r_args: &[OpRef], - allboxes: &[OpRef], - call_descr: &dyn majit_ir::descr::CallDescr, - dst: usize, - dst_bank: char, -) -> Result, DispatchError> { - if !ctx.is_authoritative_executor || r_args.len() != 2 || dst_bank != 'r' { - return Ok(None); - } - let Some(bin_op) = pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag) else { - return Ok(None); - }; - use pyre_interpreter::bytecode::BinaryOperator; - // INT_BINOP_TABLE → (OpCode, has_overflow, needs_concrete_check). - // Defer TrueDivide (int/int → float, separate helper) / Power / - // Subscr to the generic leg (`_ => None`). - let (op_code, has_overflow, needs_check) = match bin_op { - BinaryOperator::Add | BinaryOperator::InplaceAdd => (OpCode::IntAddOvf, true, false), - BinaryOperator::Subtract | BinaryOperator::InplaceSubtract => { - (OpCode::IntSubOvf, true, false) - } - BinaryOperator::Multiply | BinaryOperator::InplaceMultiply => { - (OpCode::IntMulOvf, true, false) - } - BinaryOperator::FloorDivide | BinaryOperator::InplaceFloorDivide => { - (OpCode::IntFloorDiv, false, true) - } - BinaryOperator::Remainder | BinaryOperator::InplaceRemainder => { - (OpCode::IntMod, false, true) - } - BinaryOperator::And | BinaryOperator::InplaceAnd => (OpCode::IntAnd, false, false), - BinaryOperator::Or | BinaryOperator::InplaceOr => (OpCode::IntOr, false, false), - BinaryOperator::Xor | BinaryOperator::InplaceXor => (OpCode::IntXor, false, false), - BinaryOperator::Lshift | BinaryOperator::InplaceLshift => (OpCode::IntLshift, false, true), - BinaryOperator::Rshift | BinaryOperator::InplaceRshift => (OpCode::IntRshift, false, true), - _ => return Ok(None), - }; - - // boolobject.py descr_and/or/xor: when both operands are bool the - // And/Or/Xor result is a bool (`space.newbool`), not an int. The op runs - // on the shared `intval` as for ints; only the boxing differs (picked - // below). `walker_concrete_ref_object` reads the same source as - // `walker_int_specialization_operands`, so the flag stays consistent. - let result_is_bool = matches!(op_code, OpCode::IntAnd | OpCode::IntOr | OpCode::IntXor) - && match ( - walker_concrete_ref_object(ctx, r_args[0]), - walker_concrete_ref_object(ctx, r_args[1]), - ) { - (Some(l), Some(r)) => unsafe { pyre_object::is_bool(l) && pyre_object::is_bool(r) }, - _ => false, - }; - - // Inspect the operands before executing the authentic helper. A raising - // zero-divisor arm needs the helper-produced exception as its concrete - // shadow, but records no helper call in the trace. - let Some((lhs, rhs, lhs_obj, rhs_obj, la, rb)) = - walker_int_specialization_input_operands(ctx, r_args) - else { - return Ok(None); - }; - - if matches!(op_code, OpCode::IntFloorDiv | OpCode::IntMod) && rb == 0 { - let Some(Err(exc_i64)) = walker_execute_may_force_boxed_outcome(ctx, allboxes, call_descr) - else { - return Ok(None); - }; - // The helper publishes through both the blackhole cell (drained by - // `execute_residual_call`) and the backend exception cells. The - // latter belong to compiled execution; drain the trace-time publish - // before the walk continues into the Python handler, exactly as the - // generic residual executor's Err arm does. - if let Some(cb) = crate::callbacks::try_get() { - (cb.drain_backend_jit_exc)(); - } - let exc = exc_i64 as usize as pyre_object::PyObjectRef; - let kind = pyre_object::interp_exceptions::ExcKind::ZeroDivisionError; - if !walker_recorded_builtin_raise_is_supported(exc, kind) { - return Ok(None); - } - let Some(ec) = walker_ensure_execution_context(ctx) else { - return Ok(None); - }; - - // Commit to the raising arm only after every decline. Exact-class - // guards preserve builtin dispatch; GuardTrue(rhs == 0) is the branch - // guard a bridge can invert when the divisor changes mid-loop. - let (lhs_type, lhs_descr) = crate::state::int_or_bool_unbox_type_descr(lhs_obj); - let (rhs_type, rhs_descr) = crate::state::int_or_bool_unbox_type_descr(rhs_obj); - let _lhs_raw = walker_unbox_int_typed(ctx, op_pc, lhs, lhs_type, lhs_descr)?; - walker_guard_exact_w_class(ctx, op_pc, lhs, walker_numeric_builtin_class(lhs_obj))?; - let rhs_raw = walker_unbox_int_typed(ctx, op_pc, rhs, rhs_type, rhs_descr)?; - walker_guard_exact_w_class(ctx, op_pc, rhs, walker_numeric_builtin_class(rhs_obj))?; - let rhs_zero = walker_int_eq_const(ctx, rhs_raw, 0, 1); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[rhs_zero])?; - - return Ok(Some(walker_emit_recorded_builtin_raise(ctx, ec, exc, kind))); - } - - let Some(boxed_result_i64) = walker_execute_may_force_boxed(ctx, allboxes, call_descr) else { - return Ok(None); - }; - - // intobject.py range validation (mirror the former int fast path's - // needs_concrete_check): bail to the generic leg when the bare-IR-op - // emission would be unsound (zero / INT_MIN-overflow divisor, oversized - // / overflowing shift); large right-shift folds to a const. - if needs_check { - match op_code { - OpCode::IntFloorDiv | OpCode::IntMod => { - if la == i64::MIN && rb == -1 { - return Ok(None); - } - } - OpCode::IntLshift => { - // `_lshift` enters its machine-word arm only for - // `0 <= count < LONG_BIT`. A trace recorded outside that arm - // retains the generic helper: the negative-count exception and - // the large-count/zero special case belong to the other branch. - if !(0..i64::BITS as i64).contains(&rb) { - return Ok(None); - } - let raw_value = la.wrapping_shl(rb as u32); - if (raw_value >> rb) != la { - let boxed_result_obj = boxed_result_i64 as usize as pyre_object::PyObjectRef; - if boxed_result_obj == pyre_object::PY_NULL - || !unsafe { pyre_object::is_long(boxed_result_obj) } - { - return Ok(None); - } - } - } - OpCode::IntRshift => { - // A count >= LONG_BIT (or negative) folds to 0/-1 in - // intobject.py, but that fold would be baked into the - // reused trace and be wrong for an in-range count; route it to - // the generic leg instead. An in-range recorded count is - // specialized below behind a runtime range guard. - let Ok(shift) = u32::try_from(rb) else { - return Ok(None); - }; - if shift >= i64::BITS { - return Ok(None); - } - } - _ => {} - } - } - - // pyjitpl.py handle_possible_overflow_error follows the concrete - // Add/Sub/Mul outcome. The overflowing arm mirrors intobject.py:494 - // _make_ovf2long: guard_overflow, call the elidable raw-int bigint helper - // (rbigint.py:717/788/873), and inline the W_LongObject box instead of - // falling through to the generic CallMayForceR BINARY_OP leg. - let overflows = has_overflow - && match op_code { - OpCode::IntAddOvf => la.checked_add(rb).is_none(), - OpCode::IntSubOvf => la.checked_sub(rb).is_none(), - OpCode::IntMulOvf => la.checked_mul(rb).is_none(), - _ => false, - }; - if overflows { - let boxed_result_obj = boxed_result_i64 as usize as pyre_object::PyObjectRef; - if boxed_result_obj == pyre_object::PY_NULL - || !unsafe { pyre_object::is_long(boxed_result_obj) } - { - return Ok(None); - } - } - - // --- emit the specialized IR (walker-native) --- - // bool and int share `intval`; guard each operand against its own vtable - // (BOOL_TYPE / INT_TYPE) so a bool unboxes through its own class. - let (lhs_type, lhs_descr) = crate::state::int_or_bool_unbox_type_descr(lhs_obj); - let (rhs_type, rhs_descr) = crate::state::int_or_bool_unbox_type_descr(rhs_obj); - let lhs_raw = walker_int_operand_raw(ctx, op_pc, lhs, lhs_obj, lhs_type, lhs_descr)?; - let rhs_raw = walker_int_operand_raw(ctx, op_pc, rhs, rhs_obj, rhs_type, rhs_descr)?; - - if op_code == OpCode::IntLshift { - // `ll_int_lshift_ovf` checks overflow by shifting the machine result - // back and comparing it with the input. Keep that literal shape: the - // count-range guard prevents the backend's masked shift semantics, - // then the round-trip guard selects either `_lshift`'s small-int result - // or `_ovf2long_lshift`'s bigint recovery. In particular, no backend - // overflow flag is invented for an operation that PyPy does not model - // with one. - let in_range = walker_uint_lt_const(ctx, rhs_raw, i64::BITS as i64, 1); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[in_range])?; - - let raw_value = la.wrapping_shl(rb as u32); - let raw_result = ctx - .trace_ctx - .record_op(OpCode::IntLshift, &[lhs_raw, rhs_raw]); - ctx.trace_ctx - .set_opref_concrete(raw_result, majit_ir::Value::Int(raw_value)); - let shifted_back = ctx - .trace_ctx - .record_op(OpCode::IntRshift, &[raw_result, rhs_raw]); - ctx.trace_ctx - .set_opref_concrete(shifted_back, majit_ir::Value::Int(raw_value >> rb)); - let round_trips = ctx - .trace_ctx - .record_op(OpCode::IntEq, &[shifted_back, lhs_raw]); - let overflowed = (raw_value >> rb) != la; - ctx.trace_ctx - .set_opref_concrete(round_trips, majit_ir::Value::Int((!overflowed) as i64)); - - let result = if overflowed { - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardFalse, &[round_trips])?; - walker_emit_ovf2long_box( - ctx, - op_pc, - pyre_interpreter::objspace::descroperation::jit_bigint_lshift_int_int_result - as *const (), - (lhs_raw, la), - (rhs_raw, rb), - boxed_result_i64, - )? - } else { - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[round_trips])?; - let boxed = walker_box_int(ctx, op_pc, raw_result, raw_value)?; - ctx.trace_ctx - .set_opref_concrete(boxed, box_int_concrete(raw_value, boxed_result_i64)); - boxed - }; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; - return Ok(Some(DispatchOutcome::Continue)); - } - - if overflows { - let concrete_value = match op_code { - OpCode::IntAddOvf => la.wrapping_add(rb), - OpCode::IntSubOvf => la.wrapping_sub(rb), - OpCode::IntMulOvf => la.wrapping_mul(rb), - _ => unreachable!("overflow arm requires Add/Sub/Mul"), - }; - let raw_result = ctx.trace_ctx.record_op(op_code, &[lhs_raw, rhs_raw]); - ctx.trace_ctx - .set_opref_concrete(raw_result, majit_ir::Value::Int(concrete_value)); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardOverflow, &[])?; - - let payload_fn = match op_code { - OpCode::IntAddOvf => pyre_object::longobject::jit_bigint_add_int_int as *const (), - OpCode::IntSubOvf => pyre_object::longobject::jit_bigint_sub_int_int as *const (), - OpCode::IntMulOvf => pyre_object::longobject::jit_bigint_mul_int_int as *const (), - _ => unreachable!("overflow arm requires Add/Sub/Mul"), - }; - let result = walker_emit_ovf2long_box( - ctx, - op_pc, - payload_fn, - (lhs_raw, la), - (rhs_raw, rb), - boxed_result_i64, - )?; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; - return Ok(Some(DispatchOutcome::Continue)); - } - let (raw_result, concrete_value) = match op_code { - OpCode::IntFloorDiv | OpCode::IntMod => { - walker_emit_int_div_domain_guards(ctx, op_pc, lhs_raw, rhs_raw, la, rb)?; - walker_emit_int_py_div_or_mod( - ctx, - lhs_raw, - rhs_raw, - la, - rb, - op_code == OpCode::IntFloorDiv, - ) - } - OpCode::IntRshift => { - // The machine SAR masks the count mod 64, so guard the count into - // [0, LONG_BIT) — a reused trace bails rather than shifting by - // `count & 63`. (The recorded count is < LONG_BIT here: a count - // >= LONG_BIT const-folds to 0/-1 in the needs_check block above.) - let in_range = walker_uint_lt_const(ctx, rhs_raw, i64::BITS as i64, 1); - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[in_range])?; - let r = ctx - .trace_ctx - .record_op(OpCode::IntRshift, &[lhs_raw, rhs_raw]); - (r, majit_metainterp::eval_binop_i(OpCode::IntRshift, la, rb)) - } - _ => { - let r = ctx.trace_ctx.record_op(op_code, &[lhs_raw, rhs_raw]); - (r, majit_metainterp::eval_binop_i(op_code, la, rb)) - } - }; - ctx.trace_ctx - .set_opref_concrete(raw_result, majit_ir::Value::Int(concrete_value)); - if has_overflow { - walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardNoOverflow, &[])?; - } - // A both-bool bitwise result boxes via `space.newbool` (boolobject.py: - // 74-76) so it keeps the bool type; `boxed_result_i64` is already the - // authentic W_Bool the forced residual produced. - let boxed = if result_is_bool { - match walker_newbool_guarded(ctx, op_pc, raw_result, concrete_value != 0, dst_bank)? { - Some(boxed) => boxed, - None => { - let boxed = crate::helpers::emit_trace_bool_value_from_truth( - ctx.trace_ctx, - raw_result, - false, - ); - ctx.trace_ctx.set_opref_concrete( - boxed, - majit_ir::Value::Ref(majit_ir::GcRef(boxed_result_i64 as usize)), - ); - boxed - } - } - } else { - let boxed = walker_box_int(ctx, op_pc, raw_result, concrete_value)?; - ctx.trace_ctx - .set_opref_concrete(boxed, box_int_concrete(concrete_value, boxed_result_i64)); - boxed - }; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, boxed)?; - Ok(Some(DispatchOutcome::Continue)) -} - /// rint.py `_ovf_zer` guards for a machine-int division: `int_eq(rhs,0)` → /// `guard_false` plus `(lhs==INT_MIN)&(rhs==-1)` → `guard_false`. Both must /// precede the elidable `ll_int_py_div` / `ll_int_py_mod` call so a re-used @@ -1081,6 +519,105 @@ pub(crate) fn try_walker_specialize_binary_op_long_int( Ok(Some(())) } +/// Exact-int `//` / `%` by a zero divisor, recorded as the interpreter's raise +/// rather than as the descent's materialiser call. +/// +/// `binary_value_from_tag`'s `int_floordiv` / `int_mod` bodies build their +/// `ZeroDivisionError` through `pyerror_zero_division_to_exc_object`, a +/// published `dont_look_inside` materialiser (`front/result_exc.rs` +/// `FUSED_KIND_CTORS`). A descent therefore records the instance as the result +/// of an opaque call, and an opaque call's result is a concrete object: +/// `OptVirtualize` can fold away neither it, nor the `PyTraceback` the raise +/// links onto it, nor the `sys_exc_value` save/restore around the handler. A +/// `try: x // 0 / except ZeroDivisionError:` loop then materialises all of them +/// on every iteration. `walker_emit_recorded_builtin_raise` records the same +/// construction as generated ops, which the optimizer removes when nothing +/// observes the instance. +/// +/// The divisor test is a `GUARD_TRUE(int_eq(divisor, 0))`, so a later non-zero +/// divisor side-exits to a bridge that takes the dividing arm. +#[allow(clippy::too_many_arguments)] +pub(crate) fn try_walker_specialize_binary_op_int_zero_div( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + op_tag: i64, + r_args: &[OpRef], + allboxes: &[OpRef], + call_descr: &dyn majit_ir::descr::CallDescr, + dst_bank: char, +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor || r_args.len() != 2 || dst_bank != 'r' { + return Ok(None); + } + use pyre_interpreter::bytecode::BinaryOperator; + if !matches!( + pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag), + Some( + BinaryOperator::FloorDivide + | BinaryOperator::InplaceFloorDivide + | BinaryOperator::Remainder + | BinaryOperator::InplaceRemainder + ) + ) { + return Ok(None); + } + let lhs = r_args[0]; + let rhs = r_args[1]; + let (Some(lhs_obj), Some(rhs_obj)) = ( + walker_concrete_ref_object(ctx, lhs), + walker_concrete_ref_object(ctx, rhs), + ) else { + return Ok(None); + }; + // Same exactness gate as `try_walker_specialize_truth_int`: `is_int` reads + // `ob_type`, which an `int` subclass shares, and + // `walker_numeric_builtin_class` answers with the canonical `int` for one. + unsafe { + for obj in [lhs_obj, rhs_obj] { + if !pyre_object::is_int(obj) + || pyre_object::is_bool(obj) + || !pyre_object::is_exact_builtin_instance(obj) + { + return Ok(None); + } + } + if pyre_object::w_int_get_value(rhs_obj) != 0 { + return Ok(None); + } + } + // The raising arm needs the helper-produced exception as its concrete + // shadow, but records no helper call in the trace. + let Some(Err(exc_i64)) = walker_execute_may_force_boxed_outcome(ctx, allboxes, call_descr) + else { + return Ok(None); + }; + // The helper publishes through both the blackhole cell (drained by + // `execute_residual_call`) and the backend exception cells. The latter + // belong to compiled execution; drain the trace-time publish before the + // walk continues into the Python handler. + if let Some(cb) = crate::callbacks::try_get() { + (cb.drain_backend_jit_exc)(); + } + let exc = exc_i64 as usize as pyre_object::PyObjectRef; + let kind = pyre_object::interp_exceptions::ExcKind::ZeroDivisionError; + if !walker_recorded_builtin_raise_is_supported(exc, kind) { + return Ok(None); + } + let Some(ec) = walker_ensure_execution_context(ctx) else { + return Ok(None); + }; + + // Commit to the raising arm only after every decline. + let int_type_addr = &pyre_object::pyobject::INT_TYPE as *const _ as i64; + let _lhs_raw = walker_unbox_int(ctx, op_pc, lhs, int_type_addr)?; + walker_guard_exact_w_class(ctx, op_pc, lhs, walker_numeric_builtin_class(lhs_obj))?; + let rhs_raw = walker_unbox_int(ctx, op_pc, rhs, int_type_addr)?; + walker_guard_exact_w_class(ctx, op_pc, rhs, walker_numeric_builtin_class(rhs_obj))?; + let rhs_zero = walker_int_eq_const(ctx, rhs_raw, 0, 1); + walker_emit_guard_with_snapshot(ctx, op_pc, OpCode::GuardTrue, &[rhs_zero])?; + Ok(Some(walker_emit_recorded_builtin_raise(ctx, ec, exc, kind))) +} + /// Walker-native `W_LongObject // W_IntObject` / `%` specialization for the /// `BINARY_OP` helper residual_call. /// @@ -5009,6 +4546,7 @@ fn try_walker_orthodox_load_super_attr( op_pc, sym, &sub_body, + None, "load_super_attr_value_commit", "load_super_attr_value_call_site", &[is_two_arg, self_is_cell_arg, class_slot_arg], @@ -5025,6 +4563,7 @@ fn try_walker_orthodox_load_super_attr( ConcreteValue::Ref(frame_ptr as pyre_object::PyObjectRef), ConcreteValue::Ref(w_name), ], + &[], ); let (outcome, _walk_start) = match walk { Ok(pair) => pair, @@ -7134,8 +6673,7 @@ fn walker_emit_specialised_tuple_ii( /// it always materializes the boxed bool the generic `compare_fn` would /// have produced (the retired MIFrame compare/jump fusion does not apply). /// -/// Same gate + return contract as -/// [`try_walker_specialize_binary_op_int`]. +/// Same result-writing contract as the neighbouring compare specializations. pub(crate) fn try_walker_specialize_compare_op_int( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, @@ -7491,7 +7029,7 @@ pub(crate) fn try_walker_fold_check_exc_match( /// into the operand-stack slot the guard's own resume image describes, and /// swapping the prebuilt singleton in for a recorded call result leaves it /// storing the same value. -fn walker_newbool_guarded( +pub(crate) fn walker_newbool_guarded( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, truth: OpRef, @@ -7520,6 +7058,62 @@ fn walker_newbool_guarded( Ok(Some(const_bool)) } +/// A `w_bool_from(truth)` residual met inside a descended body, folded the +/// way `space.newbool` traces: a guard on the truth and the prebuilt +/// singleton as a constant, instead of the elidable call the codewriter +/// classifies it as (`call.rs BOOL_FROM_TARGETS`), which a variable truth +/// would keep as one call per comparison. The callee is recognised by its +/// published address (`runtime_fnaddr_by_path`). A constant truth needs no +/// guard. Declines when the truth has no concrete value or the descr shape +/// is not the one-int-argument, ref-result one. +pub(crate) fn try_walker_fold_newbool_call( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + allboxes: &[OpRef], + i_args: &[OpRef], + dst: usize, + dst_bank: char, +) -> Result, DispatchError> { + if !ctx.fbw_mode.inline_subwalk || dst_bank != 'r' || i_args.len() != 1 { + return Ok(None); + } + let Some(&funcbox) = allboxes.first() else { + return Ok(None); + }; + if !funcbox.is_constant() { + return Ok(None); + } + let Some(majit_ir::Value::Int(addr)) = ctx.trace_ctx.box_value(funcbox) else { + return Ok(None); + }; + if crate::runtime_fnaddr_patch::runtime_fnaddr_by_path("pyre_object::boolobject::w_bool_from") + != Some(addr) + { + return Ok(None); + } + let truth = i_args[0]; + let Some(majit_ir::Value::Int(value)) = ctx.trace_ctx.box_value(truth) else { + return Ok(None); + }; + let observed = value != 0; + let result = if truth.is_constant() { + let result_obj = pyre_object::w_bool_from(observed); + let const_bool = ctx.trace_ctx.const_ref(result_obj as i64); + ctx.trace_ctx.set_opref_concrete( + const_bool, + majit_ir::Value::Ref(majit_ir::GcRef(result_obj as usize)), + ); + const_bool + } else { + let Some(guarded) = walker_newbool_guarded(ctx, op_pc, truth, observed, dst_bank)? else { + return Ok(None); + }; + guarded + }; + write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; + Ok(Some(())) +} + /// Does `tp` name a layout whose class overrides `is_w` with a value /// comparison? `baseobjspace::is_w` gates one branch per overriding class, /// each demanding both operands be that exact type: `int` @@ -7528,9 +7122,21 @@ fn walker_newbool_guarded( /// (`bytesobject.py:25`), `str` (`unicodeobject.py:101`) and `frozenset` /// (`setobject.py:592`). Every other class keeps the default pointer /// identity (`baseobjspace.py:246`). +/// +/// Those gates read `w_class`, this list reads `ob_type`, and the two do not +/// stand in one-to-one correspondence, so a class costs one row per layout +/// that can carry it. `int` costs two: a machine-word `W_IntObject` is +/// `INT_TYPE`, a BigInt-backed `W_LongObject` is `LONG_TYPE` +/// (`longobject.rs`), and both are born with `int`'s `w_class`, so both +/// reach the bigint comparison. The specialised arity-2 tuples also carry +/// `tuple`'s `w_class` under their own `ob_type`, but they need no row: +/// `is_w`'s tuple branch answers true only for two *empty* tuples, and a +/// length-2 layout is never empty, so `ptr_eq` already agrees for every +/// object that layout can hold. fn is_w_compares_by_value(tp: *const pyre_object::pyobject::PyType) -> bool { [ &pyre_object::pyobject::INT_TYPE as *const pyre_object::pyobject::PyType, + &pyre_object::pyobject::LONG_TYPE as *const pyre_object::pyobject::PyType, &pyre_object::pyobject::FLOAT_TYPE as *const pyre_object::pyobject::PyType, &pyre_object::pyobject::COMPLEX_TYPE as *const pyre_object::pyobject::PyType, &pyre_object::pyobject::TUPLE_TYPE as *const pyre_object::pyobject::PyType, @@ -8278,8 +7884,9 @@ pub(crate) fn try_walker_specialize_compare_op_long( /// #57 SLICE 3c: walker-native speculative float specialization for the /// `BINARY_OP` helper residual_call (oopspec `BinaryOp`), the float -/// analogue of [`try_walker_specialize_binary_op_int`]. Re-derives -/// the former float fast path's structure walker-native: per operand +/// remaining float hand fold after the exact-int fold was retired in favour +/// of [`try_walker_orthodox_binary_op`]. Re-derives the former float fast +/// path's structure walker-native: per operand /// either `guard_class FLOAT` + `getfield_gc_pure_f`, or (int operand) /// `guard_class INT` + `getfield_gc_i` + `cast_int_to_float`; then /// `float_OP` and `wrapfloat`. @@ -8288,8 +7895,8 @@ pub(crate) fn try_walker_specialize_compare_op_long( /// `FloatMul` / `FloatTrueDiv`) are specialized — Power / FloorDivide / /// Remainder have no FLOAT_* opcode and defer to the generic /// `CALL_MAY_FORCE` leg (Power lowers to a `call_may_force` + -/// `guard_no_exception` there). Tried as a fallback only after the int -/// specialization declines, so two-int operands keep int `__op__` +/// `guard_no_exception` there). Tried only after the generated exact-int +/// descent declines, so two-int operands keep the interpreter body's integer /// arithmetic. pub(crate) fn try_walker_specialize_binary_op_float( ctx: &mut WalkContext<'_, '_, Sym>, @@ -9328,6 +8935,9 @@ fn try_walker_orthodox_subscr_tuple_item( return Ok(None); } let sym = unsafe { &*sym_ptr }; + let Ok(nested_entry) = orthodox_helper_nested_entry(ctx, op_pc) else { + return Ok(None); + }; let pre_fold_pos = ctx.trace_ctx.get_trace_position(); // Only a specialisation carries its own `ob_type`, so the class guard below @@ -9356,12 +8966,14 @@ fn try_walker_orthodox_subscr_tuple_item( op_pc, sym, &sub_body, + nested_entry, "subscr_tuple_item_commit", "w_tuple_getitem_known_call_site", &[index_arg], &[ConcreteValue::Int(raw_key)], &[seq_op], &[ConcreteValue::Ref(seq_obj)], + &[], ); let (walk_outcome, _walk_start) = match walk { Ok(pair) => pair, @@ -9393,396 +9005,317 @@ fn try_walker_orthodox_subscr_tuple_item( Ok(Some(())) } -/// Descend `invert_inner`'s compiled body for `~x` on an exact builtin `int` or -/// `long`, instead of re-emitting its arms by hand. -/// -/// This is the orthodox shape: upstream traces *through* `descr_invert`, which -/// is ordinary RPython, rather than carrying a fold per operand type. What -/// makes it worth entering here is coverage, not the emitted shape -- the -/// hand-written `unary_invert_int` answers the exact-`int` operand only, and -/// `~` on a `long` reaches no fold at all today (measured: the fold is -/// consulted for both and fires for one). The body's own arms cover both, so -/// descending it covers the second without a second fold. -/// -/// `invert` itself cannot be entered. Its `__invert__` override probe is -/// `dont_look_inside` and is the second operation the body executes, and its -/// bool slot raises a deprecation warning, which reaches `lookup_exc_class`. -/// `invert_inner` is that body past both. The guards emitted here are what -/// let the caller skip them: `bool` carries its own type, and an `int` or -/// `long` subclass keeps the builtin `ob_type` but retags `w_class` and may -/// define `__invert__`, so both must side-exit to the residual, which still -/// runs the whole of `invert`. -pub(crate) fn try_walker_orthodox_unary_invert( +/// An operator's descent: which body to enter and how the trace names the +/// site. +struct HelperDescent { + /// Canonical path of the operator's own body -- `descroperation::pos`, + /// `opcode_ops::binary_value_from_tag` -- not a split of it. + path: &'static str, + commit_label: &'static str, + call_site_label: &'static str, + decline_tag: &'static str, +} + +/// The `BINARY_OP` helper itself: the operator tag is a trace-time constant, +/// so its `match` folds and only the selected operator's body is traced. +const BINARY_OP_DESCENT: HelperDescent = HelperDescent { + path: "pyre_interpreter::opcode_ops::binary_value_from_tag", + commit_label: "binary_op_commit", + call_site_label: "binary_op_call_site", + decline_tag: "BINARY-OP-SUBWALK", +}; + +const COMPARE_OP_DESCENT: HelperDescent = HelperDescent { + path: "pyre_interpreter::opcode_ops::compare_value_from_tag", + commit_label: "compare_op_commit", + call_site_label: "compare_op_call_site", + decline_tag: "COMPARE-OP-SUBWALK", +}; + +/// Descend an operator's body whole instead of re-emitting an arm of it by +/// hand. +/// +/// Upstream traces *through* `descr_pos`, `descr_add` and their siblings; +/// the guards that select the arm are the body's own. Here they are too: +/// the class tests the operator opens with read `ob_type`, which the +/// codewriter emits as `guard_class`, and its override probes promote +/// `w_class` (`descroperation.rs try_numeric_unaryop_override`, +/// `needs_numeric_binop_dispatch`), which records a `guard_value`. Nothing +/// about the operands is asserted at this call site, so a receiver the body +/// handles differently on the next entry side-exits at one of those guards +/// and re-runs the operator in the residual. +/// +/// The caller decides *whether* to descend, and that is a policy, not a +/// guard: an operand that is not an exact builtin instance takes an override +/// arm, which calls Python, and a sub-walk that executes a call and then +/// declines has run it twice. Until a mid-descent decline rewinds such an +/// effect, that operand goes to the residual, unguarded. +/// +/// `ref_args` pairs each operand box with its concrete object; `int_args` +/// carries constant-bank operands (an operator tag) the same way. A body +/// that raises declines (see the `SubRaise` arm). +fn try_walker_orthodox_descent( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, - operand: OpRef, + int_args: &[(OpRef, i64)], + ref_args: &[(OpRef, pyre_object::PyObjectRef)], dst: usize, dst_bank: char, -) -> Result, DispatchError> { - let Some(operand_obj) = walker_concrete_ref_object(ctx, operand) else { - return Ok(None); - }; - // SAFETY: `operand_obj` is a live concrete `PyObjectRef` from the walker - // shadow. - let is_long = unsafe { pyre_object::is_long(operand_obj) }; - let admitted = unsafe { - !pyre_object::is_bool(operand_obj) - && (is_long || pyre_object::is_int(operand_obj)) - && pyre_object::is_exact_builtin_instance(operand_obj) - }; - if !admitted { - return Ok(None); - } - // SAFETY: as above. - // - // This cannot decline for an admitted operand. `walker_exact_builtin_class` - // answers `None` for an exact builtin whose `w_class` is null, and the only - // objects born that way are the read-only singletons (`True`, `False`, - // `None`, `Ellipsis`, `NotImplemented`) -- every other builtin is born - // carrying `get_instantiate(ob_type)`. None of those five survives the - // admission above: the first two are `bool`, the rest are not `int`. - let Some(operand_class) = (unsafe { walker_exact_builtin_class(operand_obj) }) else { - return Ok(None); + descent: &HelperDescent, +) -> Result, DispatchError> { + // Resolve every possible decline before recording anything. Each + // decline names itself under `PYRE_FBW_DEBUG_ABORT` so a `consulted=1 + // fired=0` census line can be attributed without a rebuild. + let decline = |why: &str| { + if fbw_debug_abort_enabled() { + eprintln!("[decline-why] {}-{why} pc={op_pc}", descent.decline_tag); + } + Ok(None) }; - - // Resolve every possible decline before recording a guard. - let Some(jc_arc) = crate::jitcode_runtime::invert_inner_jitcode() else { - return Ok(None); + let Some(jc_arc) = crate::jitcode_runtime::pathed_jitcode_cached(descent.path) else { + return decline("NO-JITCODE"); }; let Some(sub_body) = sub_jitcode_body_by_index(jc_arc.index()) else { - return Ok(None); + return decline("NO-SUB-BODY"); }; let sym_ptr = ctx.fbw_mode.snapshot_sym; if sym_ptr.is_null() { - return Ok(None); + return decline("NO-SYM"); } // SAFETY: set for the lifetime of the enclosing full-body walk. if unsafe { (&*sym_ptr).jitcode().is_null() } { - return Ok(None); + return decline("SYM-NO-JITCODE"); } let sym = unsafe { &*sym_ptr }; - let pre_fold_pos = ctx.trace_ctx.get_trace_position(); - let type_addr = if is_long { - &pyre_object::pyobject::LONG_TYPE as *const _ as i64 - } else { - &pyre_object::pyobject::INT_TYPE as *const _ as i64 + let Ok(nested_entry) = orthodox_helper_nested_entry(ctx, op_pc) else { + return decline("NESTED-ENTRY"); }; - walker_guard_class(ctx, op_pc, operand, type_addr)?; - walker_guard_exact_w_class(ctx, op_pc, operand, operand_class)?; - ctx.trace_ctx.set_opref_concrete( - operand, - majit_ir::Value::Ref(majit_ir::GcRef(operand_obj as usize)), - ); + + let pre_fold_pos = ctx.trace_ctx.get_trace_position(); + for &(operand, operand_obj) in ref_args { + ctx.trace_ctx.set_opref_concrete( + operand, + majit_ir::Value::Ref(majit_ir::GcRef(operand_obj as usize)), + ); + } + let int_oprefs: Vec = int_args.iter().map(|&(opref, _)| opref).collect(); + let int_concretes: Vec = int_args + .iter() + .map(|&(_, value)| ConcreteValue::Int(value)) + .collect(); + let ref_oprefs: Vec = ref_args.iter().map(|&(opref, _)| opref).collect(); + let ref_concretes: Vec = ref_args + .iter() + .map(|&(_, obj)| ConcreteValue::Ref(obj)) + .collect(); let walk = run_orthodox_helper_subwalk( ctx, op_pc, sym, &sub_body, - "unary_invert_commit", - "invert_inner_call_site", + nested_entry, + descent.commit_label, + descent.call_site_label, + &int_oprefs, + &int_concretes, + &ref_oprefs, + &ref_concretes, &[], - &[], - &[operand], - &[ConcreteValue::Ref(operand_obj)], ); let (walk_outcome, _walk_start) = match walk { - // The body reached a helper this build did not lower. `invert_inner` - // is a pure read on both admitted arms, so nothing is committed -- - // cut the tentative IR, with its snapshots, and let the residual serve - // the operator. The two guards above are emitted with snapshots - // attached and name the discarded operation namespace, so leaving them - // behind would expose stale boxes to a later remap. + // The body reached a helper this build did not lower. The arms an + // exact builtin takes are reads that allocate at most their result, + // so nothing is committed -- cut the tentative IR, with its + // snapshots, and let the residual (or the fold behind this descent) + // serve the operator. Err(DispatchError::OrthodoxSubWalkTraceUnsupported { pc, .. }) => { if fbw_debug_abort_enabled() { - eprintln!("[decline-why] UNARY-INVERT-SUBWALK pc={pc}"); + eprintln!("[decline-why] {} pc={pc}", descent.decline_tag); } ctx.trace_ctx.cut_trace_with_snapshots(pre_fold_pos); ctx.trace_ctx.heap_cache_mut().reset(); return Ok(None); } Ok(pair) => pair, - Err(error) => return Err(error), + Err(error) => { + if fbw_debug_abort_enabled() { + eprintln!( + "[decline-why] {}-ERR pc={op_pc} error={}", + descent.decline_tag, + error.variant_name() + ); + } + return Err(error); + } }; let result = match walk_outcome { DispatchOutcome::SubReturn { result } => finish_inline_callee_return(ctx, result) .ok_or(DispatchError::UnexpectedVoidSubReturn { pc: op_pc })?, + // `front::result_exc::fuse_kind_ctor_raise` removes the Rust + // `PyError` aggregate from supported literal-message raise paths and + // materialises the interpreter's `W_BaseException` through one opaque + // residual. That is the exception value the ordinary inline-callee + // machinery propagates too, so preserve the sub-walk's `SubRaise` + // instead of rolling it back to a hand-written operator fold. + raised @ DispatchOutcome::SubRaise { .. } => return Ok(Some(raised)), _ => return Err(DispatchError::UnexpectedVoidSubReturn { pc: op_pc }), }; write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; - Ok(Some(())) + Ok(Some(DispatchOutcome::Continue)) } -/// Descend `neg_inner`'s compiled body for `-x` on an exact builtin `int` or -/// `long`, instead of re-emitting its integer arm by hand. -/// -/// The same orthodox shape as [`try_walker_orthodox_unary_invert`], and for the -/// same reason: upstream traces *through* `descr_neg`, which is ordinary -/// RPython. -/// -/// The descent also owns the `INT_MIN` promotion. `rbigint.neg` stays an -/// `EF_ELIDABLE_OR_MEMORYERROR` residual, while `W_LongObject.__init__` lowers -/// to `NewWithVtable` plus its ordinary `w_class` and `value` field writes. -/// This is the same allocation/body split PyPy's rtyper and metainterp expose; -/// no unary-negative manual fold remains beside it. -/// -/// `neg` itself cannot be entered: its `__neg__` override probe is -/// `dont_look_inside` and is the second operation the body executes. -/// `neg_inner` is that body past it. Unlike `invert`, `neg` has no bool slot to -/// step over, so the split leaves the probe alone. -/// -/// The guards emitted here are what let the caller skip the probe: an `int` or -/// `long` subclass keeps the builtin `ob_type` but retags `w_class` and may -/// define `__neg__`, so it must side-exit to the residual, which still runs the -/// whole of `neg`. `bool` takes the same `neg_inner` integer arm and cannot -/// be subclassed, so its dedicated class guard is sufficient; unlike an -/// ordinary int it needs no `w_class` version guard. -pub(crate) fn try_walker_orthodox_unary_negative( +/// `a OP b`: descend `binary_value_from_tag` with the operator tag as a +/// constant. See [`try_walker_orthodox_descent`]. +/// +/// Policy: both operands must be concrete exact builtin machine ints (int, +/// bool). Exactness keeps the override arms, which call Python, out +/// of the sub-walk; the numeric restriction keeps out the sequence arms, +/// whose in-place forms (`list += list`) mutate the receiver before any +/// later decline could rewind them. +/// +/// An in-place tag is descended as its plain operator. The body routes +/// tags 13..=24 through the residual `binary_value` (the `__iadd__` probe +/// comes first there), which the sub-walk cannot enter -- every `t += x` +/// declined, 129 cuts in one `synth/trace_segmenting_over_limit_retry` run. +/// An exact builtin numeric has no in-place special, so `binary_value` +/// reaches the same `add`/`sub`/... arm the plain tag selects directly. +pub(crate) fn try_walker_orthodox_binary_op( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, - operand: OpRef, + op_tag: i64, + tag: OpRef, + r_args: &[OpRef], dst: usize, dst_bank: char, -) -> Result, DispatchError> { - let Some(operand_obj) = walker_concrete_ref_object(ctx, operand) else { - return Ok(None); - }; - // SAFETY: `operand_obj` is a live concrete `PyObjectRef` from the walker - // shadow. - let is_bool = unsafe { pyre_object::is_bool(operand_obj) }; - let is_long = unsafe { pyre_object::is_long(operand_obj) }; - let admitted = unsafe { - is_bool - || ((is_long || pyre_object::is_int(operand_obj)) - && pyre_object::is_exact_builtin_instance(operand_obj)) - }; - if !admitted { +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor || r_args.len() != 2 || dst_bank != 'r' { return Ok(None); } - // SAFETY: as above. + // `//` and `%` descend since `int_floordiv` / `int_mod` compute the + // floor result through the `#[oopspec("int.py_div")]` / + // `int.py_mod` twins of rint.py's `ll_int_py_div` / `ll_int_py_mod`: + // the body records the same elidable call the fold did, so the + // optimizer's constant-divisor strength reduction applies to both. + // Before that the body's `/` / `%` lowered to the truncating + // `_ll_2_int_*` residual plus the sign-correction branch, measured + // 63 → 79 ops on `i // (i % 3)`. // - // The bool singletons deliberately carry no user-class stamp. They also - // cannot have subclasses, so the `walker_guard_class` check against - // `BOOL_TYPE` proves the complete dispatch decision. Int and long retain - // the exact-class guard because their subclasses can override `__neg__`. - let operand_class = if is_bool { - None - } else { - let Some(class) = (unsafe { walker_exact_builtin_class(operand_obj) }) else { - return Ok(None); - }; - Some(class) - }; - - // Resolve every possible decline before recording a guard. - // - // `-INT_MIN` is the one operand `descr_neg` promotes, and it belongs to - // [`try_walker_specialize_unary_negative_int`] rather than to this walk. - // Walking the promotion records the two `rbigint` calls that build the - // `2**63` long; both survive optimization as `CallPureR` short boxes and - // the second one's result is carried across the `LABEL` as a loop - // argument, so a later pure call reading that long -- `compare_op_long`'s - // `jit_bigint_cmp` -- has no constant argument and is re-emitted into the - // body as an impure `CallI` once per iteration. The fold pins the unboxed - // operand with `guard_value` instead, and then the whole chain is - // constant: it exports no short box at all and the comparison folds away - // (`unary_negative.py main_int_min`, 20 ops / 9 guards -> 17 / 8). - if let Some((x, _)) = walker_unary_int_operand(ctx, operand) - && x == i64::MIN - { - return Ok(None); - } - let Some(jc_arc) = crate::jitcode_runtime::neg_inner_jitcode() else { - return Ok(None); + // `**` stays with the fold: `float_pow` reaches `f64::is_infinite` + // and the `float` constructor, neither lowered, so the sub-walk declined + // at every `(n + 1.25) ** 1.5` of `synth/wasm_ca_trampoline_decline`. + use pyre_interpreter::bytecode::BinaryOperator as B; + let plain = match pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag) { + Some(B::Add | B::InplaceAdd) => B::Add, + Some(B::Subtract | B::InplaceSubtract) => B::Subtract, + Some(B::Multiply | B::InplaceMultiply) => B::Multiply, + Some(B::FloorDivide | B::InplaceFloorDivide) => B::FloorDivide, + Some(B::Remainder | B::InplaceRemainder) => B::Remainder, + Some(B::TrueDivide | B::InplaceTrueDivide) => B::TrueDivide, + Some(B::Lshift | B::InplaceLshift) => B::Lshift, + Some(B::Rshift | B::InplaceRshift) => B::Rshift, + Some(B::And | B::InplaceAnd) => B::And, + Some(B::Or | B::InplaceOr) => B::Or, + Some(B::Xor | B::InplaceXor) => B::Xor, + _ => return Ok(None), }; - let Some(sub_body) = sub_jitcode_body_by_index(jc_arc.index()) else { + let Some(plain_tag) = pyre_interpreter::runtime_ops::binary_op_tag(plain) else { return Ok(None); }; - let sym_ptr = ctx.fbw_mode.snapshot_sym; - if sym_ptr.is_null() { - return Ok(None); - } - // SAFETY: set for the lifetime of the enclosing full-body walk. - if unsafe { (&*sym_ptr).jitcode().is_null() } { - return Ok(None); + let mut operands = [(OpRef::NONE, std::ptr::null_mut()); 2]; + for (slot, &operand) in operands.iter_mut().zip(r_args) { + let Some(obj) = walker_concrete_ref_object(ctx, operand) else { + return Ok(None); + }; + // SAFETY: `obj` is a live concrete `PyObjectRef` from the walker + // shadow. + // + // No `long`: its arms run rbigint, which this build does not lower, + // so the sub-walk declined every time (137 cuts in one + // `synth/wasm_ca_trampoline_decline` run, each a rewound trace). + // No `float` either: `w_float_new` allocates through a synthetic + // transparent `PyObject` constructor with no host symbol, and the + // sub-walk declined at it on every float arm (the unary `neg` + // descent declines there the same way). + let admitted = unsafe { + pyre_object::is_exact_builtin_instance(obj) + && (pyre_object::is_int(obj) || pyre_object::is_bool(obj)) + }; + if !admitted { + return Ok(None); + } + *slot = (operand, obj); } - let sym = unsafe { &*sym_ptr }; - - let pre_fold_pos = ctx.trace_ctx.get_trace_position(); - let type_addr = if is_bool { - &pyre_object::pyobject::BOOL_TYPE as *const _ as i64 - } else if is_long { - &pyre_object::pyobject::LONG_TYPE as *const _ as i64 + let tag = if plain_tag == op_tag { + tag } else { - &pyre_object::pyobject::INT_TYPE as *const _ as i64 + ctx.trace_ctx.const_int(plain_tag) }; - walker_guard_class(ctx, op_pc, operand, type_addr)?; - if let Some(operand_class) = operand_class { - walker_guard_exact_w_class(ctx, op_pc, operand, operand_class)?; - } - ctx.trace_ctx.set_opref_concrete( - operand, - majit_ir::Value::Ref(majit_ir::GcRef(operand_obj as usize)), - ); - - let walk = run_orthodox_helper_subwalk( + try_walker_orthodox_descent( ctx, op_pc, - sym, - &sub_body, - "unary_negative_commit", - "neg_inner_call_site", - &[], - &[], - &[operand], - &[ConcreteValue::Ref(operand_obj)], - ); - let (walk_outcome, _walk_start) = match walk { - // The body reached a helper this build did not lower. Both admitted - // arms of `neg_inner` are pure reads that allocate their result, so - // nothing is committed -- cut the tentative IR, with its snapshots, and - // let the residual serve the operator. The two guards above are emitted - // with snapshots attached and name the discarded operation namespace, - // so leaving them behind would expose stale boxes to a later remap. - Err(DispatchError::OrthodoxSubWalkTraceUnsupported { pc, .. }) => { - if fbw_debug_abort_enabled() { - eprintln!("[decline-why] UNARY-NEGATIVE-SUBWALK pc={pc}"); - } - ctx.trace_ctx.cut_trace_with_snapshots(pre_fold_pos); - ctx.trace_ctx.heap_cache_mut().reset(); - return Ok(None); - } - Ok(pair) => pair, - Err(error) => return Err(error), - }; - let result = match walk_outcome { - DispatchOutcome::SubReturn { result } => finish_inline_callee_return(ctx, result) - .ok_or(DispatchError::UnexpectedVoidSubReturn { pc: op_pc })?, - _ => return Err(DispatchError::UnexpectedVoidSubReturn { pc: op_pc }), - }; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; - Ok(Some(())) + &[(tag, plain_tag)], + &operands, + dst, + dst_bank, + &BINARY_OP_DESCENT, + ) } -/// Descend `pos_inner`'s compiled body for `+x` on an exact builtin `int` or -/// `long`, instead of re-emitting its identity arm by hand. -/// -/// The same orthodox shape as [`try_walker_orthodox_unary_invert`] and -/// [`try_walker_orthodox_unary_negative`]. Upstream traces *through* -/// `descr_pos`. The exact-int and exact-long arms are identity: they return -/// the operand, matching `_self_unaryop('pos')`. -/// -/// `pos` itself cannot be entered: its `__pos__` override probe is -/// `dont_look_inside` and is the second operation the body executes. -/// `pos_inner` is that body past it. Unlike `invert`, `pos` has no bool slot -/// to step over, so the split leaves the probe alone. -/// -/// The guards emitted here are what let the caller skip the probe: an `int` or -/// `long` subclass keeps the builtin `ob_type` but retags `w_class` and may -/// define `__pos__`, so it must side-exit to the residual, which still runs the -/// whole of `pos`. `bool` is excluded because `+True` is a rewrapping to a -/// plain int, not identity, and because [`walker_exact_builtin_class`] has no -/// value to pin on a singleton. -pub(crate) fn try_walker_orthodox_unary_positive( +/// `COMPARE_OP` on two exact builtin machine ints (`int`, `bool`): descend +/// `compare_value_from_tag` → `compare` → `compare_slot` → `int_lt` and its +/// siblings instead of re-emitting the arm by hand +/// (`try_walker_specialize_compare_op_int`). See +/// [`try_walker_orthodox_binary_op`] for the operand policy; the body's +/// override probe is promoted away for such a pair +/// (`descroperation.rs compare`), and the `bool`-vs-`int` subtype ordering +/// it keeps is decided on the promoted classes. +/// +/// Tags 0..=5 are the six rich comparisons. `in` / `not in` (6, 7) take +/// `contains`, `is` / `is_not` (8, 9) have their own fold, and +/// CHECK_EXC_MATCH (10) its own; none of them is this body's comparison. +pub(crate) fn try_walker_orthodox_compare_op( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, - operand: OpRef, + op_tag: i64, + tag: OpRef, + r_args: &[OpRef], dst: usize, dst_bank: char, -) -> Result, DispatchError> { - let Some(operand_obj) = walker_concrete_ref_object(ctx, operand) else { - return Ok(None); - }; - // SAFETY: `operand_obj` is a live concrete `PyObjectRef` from the walker - // shadow. - let is_long = unsafe { pyre_object::is_long(operand_obj) }; - let admitted = unsafe { - !pyre_object::is_bool(operand_obj) - && (is_long || pyre_object::is_int(operand_obj)) - && pyre_object::is_exact_builtin_instance(operand_obj) - }; - if !admitted { - return Ok(None); - } - // SAFETY: as above. - // - // This cannot decline for an admitted operand, by the argument - // [`try_walker_orthodox_unary_invert`] gives: the only exact builtins born - // with a null `w_class` are the five read-only singletons, and the - // admission above already rejects every one of them. - let Some(operand_class) = (unsafe { walker_exact_builtin_class(operand_obj) }) else { - return Ok(None); - }; - - // Resolve every possible decline before recording a guard. - let Some(jc_arc) = crate::jitcode_runtime::pos_inner_jitcode() else { - return Ok(None); - }; - let Some(sub_body) = sub_jitcode_body_by_index(jc_arc.index()) else { - return Ok(None); - }; - let sym_ptr = ctx.fbw_mode.snapshot_sym; - if sym_ptr.is_null() { - return Ok(None); - } - // SAFETY: set for the lifetime of the enclosing full-body walk. - if unsafe { (&*sym_ptr).jitcode().is_null() } { - return Ok(None); - } - let sym = unsafe { &*sym_ptr }; - - let pre_fold_pos = ctx.trace_ctx.get_trace_position(); - let type_addr = if is_long { - &pyre_object::pyobject::LONG_TYPE as *const _ as i64 - } else { - &pyre_object::pyobject::INT_TYPE as *const _ as i64 - }; - walker_guard_class(ctx, op_pc, operand, type_addr)?; - walker_guard_exact_w_class(ctx, op_pc, operand, operand_class)?; - ctx.trace_ctx.set_opref_concrete( - operand, - majit_ir::Value::Ref(majit_ir::GcRef(operand_obj as usize)), - ); - - let walk = run_orthodox_helper_subwalk( - ctx, - op_pc, - sym, - &sub_body, - "unary_positive_commit", - "pos_inner_call_site", - &[], - &[], - &[operand], - &[ConcreteValue::Ref(operand_obj)], - ); - let (walk_outcome, _walk_start) = match walk { - // The body reached a helper this build did not lower. Both admitted - // arms of `pos_inner` are identity reads, so nothing is committed -- - // cut the tentative IR, with its snapshots, and let the residual (or - // the identity fold behind this descent) serve the operator. The two - // guards above are emitted with snapshots attached and name the - // discarded operation namespace, so leaving them behind would expose - // stale boxes to a later remap. - Err(DispatchError::OrthodoxSubWalkTraceUnsupported { pc, .. }) => { - if fbw_debug_abort_enabled() { - eprintln!("[decline-why] UNARY-POSITIVE-SUBWALK pc={pc}"); - } - ctx.trace_ctx.cut_trace_with_snapshots(pre_fold_pos); - ctx.trace_ctx.heap_cache_mut().reset(); - return Ok(None); - } - Ok(pair) => pair, - Err(error) => return Err(error), - }; - let result = match walk_outcome { - DispatchOutcome::SubReturn { result } => finish_inline_callee_return(ctx, result) - .ok_or(DispatchError::UnexpectedVoidSubReturn { pc: op_pc })?, - _ => return Err(DispatchError::UnexpectedVoidSubReturn { pc: op_pc }), - }; - write_residual_call_result_to_dst(ctx, op_pc, dst, dst_bank, result)?; - Ok(Some(())) +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor + || r_args.len() != 2 + || dst_bank != 'r' + || !(0..=5).contains(&op_tag) + { + return Ok(None); + } + let mut operands = [(OpRef::NONE, std::ptr::null_mut()); 2]; + for (slot, &operand) in operands.iter_mut().zip(r_args) { + let Some(obj) = walker_concrete_ref_object(ctx, operand) else { + return Ok(None); + }; + // SAFETY: `obj` is a live concrete `PyObjectRef` from the walker + // shadow. + let admitted = unsafe { + pyre_object::is_exact_builtin_instance(obj) + && (pyre_object::is_int(obj) || pyre_object::is_bool(obj)) + }; + if !admitted { + return Ok(None); + } + *slot = (operand, obj); + } + try_walker_orthodox_descent( + ctx, + op_pc, + &[(tag, op_tag)], + &operands, + dst, + dst_bank, + &COMPARE_OP_DESCENT, + ) } /// `s[i]` on an exact `str` with an exact machine-`int` index: emit the @@ -10154,373 +9687,6 @@ pub(crate) fn try_walker_specialize_builtin_dict_get( walker_emit_exact_dict_hit(ctx, op.pc, dict_op, r_args[2], dict, hit, dst, 'r') } -/// Where the guarded receiver's length comes from — the `length()` body each -/// layout has upstream. -#[derive(Clone, Copy)] -enum BuiltinLenSource { - /// `W_ListObject.length()` → `strategy.length` (rlist.py). Carries the - /// storage-strategy id the read is guarded on. - ListStrategy(i64), - /// `EmptyListStrategy.length()` returns zero (`listobject.py`). - /// The strategy still needs a guard because a reused list may transition - /// to typed or object storage after tracing. - EmptyList, - /// `W_UnicodeObject.len` → `bh_unicodelen`; no storage strategy. - StrField, - /// `W_BytesObject.len` — `bytesobject.py` answers `len(self._value)` off - /// the RPython string; pyre precomputes that count into a field, so the - /// read is the same shape as [`BuiltinLenSource::StrField`]. - BytesField, - /// `W_BytearrayObject.length` — `bytearrayobject.py`'s `_len` reads the - /// length off the RPython list in `self._data`; pyre mirrors that count - /// into a field. Mutable, unlike [`BuiltinLenSource::BytesField`]. - BytearrayField, - /// `W_SetObject.len` — `setobject.py W_BaseSetObject.length` answers - /// `self.strategy.length(self)`; pyre keeps the count on the body, so the - /// read is the same shape as [`BuiltinLenSource::BytearrayField`]. - SetField, - /// `tupleobject.py` carries no separate length field, so the length is - /// `arraylen_gc(wrappeditems)`. - TupleArrayLen, - /// `functional.py W_Range.descr_len` returns the precomputed - /// wrapped `self.w_length` field unchanged. - RangeField, - /// `specialisedtupleobject.py length()` returns the constant - /// `typelen`. - PairArity, -} - -/// `len(x)` on an exact canonical `W_ListObject` / `W_UnicodeObject` / -/// `W_BytesObject` / `W_BytearrayObject` / `W_SetObject` (as either `set` or -/// `frozenset`) / `W_TupleObject` / `W_Range`, or on an arity-2 tuple -/// specialisation: -/// lower the opaque `bh_call_fn(len_builtin, PY_NULL, x)` residual to the -/// inline length read the meta-tracer produces upstream -/// (descroperation.py `_len`): `guard_value(callable)` + -/// `guard_class` + exact `w_class` guard + the [`BuiltinLenSource`] read + -/// `wrapint`. The exact `w_class` guard is required because a SUBCLASS shares -/// `ob_type == &LIST_TYPE`/`&STR_TYPE`/`&TUPLE_TYPE` but may override `__len__` -/// (`baseobjspace::len` dispatches `subclass_special_override`); it -/// side-exits to the generic residual. -/// -/// Returns `None` (fall through to the generic residual, SAFE) for any -/// other shape: non-list/str/tuple arg, a subclass, a bound receiver, or wrong -/// arity. `dict` is one of those: `W_DictObject` carries no length word at -/// all -- `dictmultiobject.py length` goes through the strategy to -/// `len(unerase(dstorage))`, and pyre's storage is an `IndexMap` whose count -/// is not a field the trace can read. -pub(crate) fn try_walker_specialize_builtin_len( - ctx: &mut WalkContext<'_, '_, Sym>, - code: &[u8], - op: &DecodedOp, - r_args: &[OpRef], - dst: usize, -) -> Result, DispatchError> { - // Plain `bh_call_fn(callable, PY_NULL, arg)` shape only. - if r_args.len() != 3 { - return Ok(None); - } - let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); - let ( - ConcreteValue::Ref(concrete_callable), - ConcreteValue::Ref(null_or_self), - ConcreteValue::Ref(list_obj), - ) = (arg_concretes[0], arg_concretes[1], arg_concretes[2]) - else { - return Ok(None); - }; - // A non-null `null_or_self` is a bound receiver `bh_call_fn_impl` - // prepends as arg0 — not a plain `len(x)` call. - if concrete_callable.is_null() || !null_or_self.is_null() || list_obj.is_null() { - return Ok(None); - } - if !pyre_interpreter::builtins::is_builtin_len_function(concrete_callable) { - return Ok(None); - } - // Exact canonical list / str / tuple / range, or one of the arity-2 tuple - // specialisations. `arg_type_addr` pins the `guard_class` target; - // `exact_w_class` is the subclass-`__len__` guard (see the doc comment), - // absent for a specialisation because only `makespecialisedtuple2` builds - // that `ob_type` and always with the canonical tuple `w_class`, so the - // class guard alone already excludes every subclass instance. - let (arg_type_addr, exact_w_class, len_source, concrete_len) = unsafe { - let ob_type = (*list_obj).ob_type; - let w_class = (*list_obj).w_class; - if std::ptr::eq(ob_type, &pyre_object::pyobject::LIST_TYPE) { - let exact = pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::LIST_TYPE); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - let len_source = if pyre_object::w_list_uses_int_storage(list_obj) { - BuiltinLenSource::ListStrategy( - pyre_object::listobject::ListStrategy::Integer as i64, - ) - } else if pyre_object::w_list_uses_float_storage(list_obj) { - BuiltinLenSource::ListStrategy(pyre_object::listobject::ListStrategy::Float as i64) - } else if pyre_object::w_list_uses_object_storage(list_obj) { - BuiltinLenSource::ListStrategy(pyre_object::listobject::ListStrategy::Object as i64) - } else if pyre_object::w_list_uses_empty_storage(list_obj) { - BuiltinLenSource::EmptyList - } else { - return Ok(None); - }; - ( - &pyre_object::pyobject::LIST_TYPE as *const _ as i64, - Some(exact), - len_source, - pyre_object::w_list_len(list_obj), - ) - } else if std::ptr::eq(ob_type, &pyre_object::pyobject::STR_TYPE) { - let exact = pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::STR_TYPE); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - ( - &pyre_object::pyobject::STR_TYPE as *const _ as i64, - Some(exact), - BuiltinLenSource::StrField, - pyre_object::w_str_len(list_obj), - ) - } else if std::ptr::eq(ob_type, &pyre_object::bytesobject::BYTES_TYPE) { - let exact = - pyre_object::pyobject::get_instantiate(&pyre_object::bytesobject::BYTES_TYPE); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - ( - &pyre_object::bytesobject::BYTES_TYPE as *const _ as i64, - Some(exact), - BuiltinLenSource::BytesField, - pyre_object::bytesobject::w_bytes_len(list_obj), - ) - } else if std::ptr::eq(ob_type, &pyre_object::bytearrayobject::BYTEARRAY_TYPE) { - let exact = pyre_object::pyobject::get_instantiate( - &pyre_object::bytearrayobject::BYTEARRAY_TYPE, - ); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - ( - &pyre_object::bytearrayobject::BYTEARRAY_TYPE as *const _ as i64, - Some(exact), - BuiltinLenSource::BytearrayField, - pyre_object::bytearrayobject::w_bytearray_len(list_obj), - ) - } else if std::ptr::eq(ob_type, &pyre_object::setobject::SET_TYPE) - || std::ptr::eq(ob_type, &pyre_object::setobject::FROZENSET_TYPE) - { - // Two types over one `W_SetObject` body, so the only thing the two - // differ in is which of them the class guard pins — and `ob_type` - // is already the one that matched. - let exact = pyre_object::pyobject::get_instantiate(&*ob_type); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - ( - ob_type as i64, - Some(exact), - BuiltinLenSource::SetField, - pyre_object::setobject::w_set_len(list_obj), - ) - } else if std::ptr::eq(ob_type, &pyre_object::pyobject::TUPLE_TYPE) { - let exact = pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::TUPLE_TYPE); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - ( - &pyre_object::pyobject::TUPLE_TYPE as *const _ as i64, - Some(exact), - BuiltinLenSource::TupleArrayLen, - pyre_object::w_tuple_len(list_obj), - ) - } else if std::ptr::eq(ob_type, &pyre_object::functional::RANGE_TYPE) { - let exact = - pyre_object::pyobject::get_instantiate(&pyre_object::functional::RANGE_TYPE); - if !std::ptr::eq(w_class, exact) { - return Ok(None); - } - let Some(concrete_len) = pyre_object::functional::w_range_length_i64(list_obj) else { - return Ok(None); - }; - let Ok(concrete_len) = usize::try_from(concrete_len) else { - return Ok(None); - }; - ( - &pyre_object::functional::RANGE_TYPE as *const _ as i64, - Some(exact), - BuiltinLenSource::RangeField, - concrete_len, - ) - } else if specialised_pair_kind(ob_type).is_some() { - ( - ob_type as i64, - None, - BuiltinLenSource::PairArity, - pyre_object::w_tuple_len(list_obj), - ) - } else { - return Ok(None); - } - }; - - // Authentic boxed result, produced on the plain eval loop exactly as - // the skipped residual would (len on an exact list is side-effect-free). - let boxed_result = { - let _plain_guard = pyre_interpreter::call::force_plain_eval(); - pyre_interpreter::call::call_function_impl_result(concrete_callable, &[list_obj]) - }; - let Ok(boxed_result) = boxed_result else { - return Ok(None); - }; - - // --- emit the specialized IR (walker-native) --- - // Pin the callable identity (LOAD_GLOBAL `len` is usually already a - // constant via the namespace cell fold). - let callable_op = r_args[0]; - if !callable_op.is_constant() { - let expected = ctx.trace_ctx.const_ref(concrete_callable as i64); - ctx.trace_ctx - .record_guard(OpCode::GuardValue, &[callable_op, expected], 0); - walker_capture_snapshot_for_last_guard(ctx, op.pc)?; - ctx.trace_ctx - .heap_cache_mut() - .replace_box(callable_op, expected); - } - let list_op = r_args[2]; - // guard_class (skip when class already known / operand is constant). - if !list_op.is_constant() && !ctx.trace_ctx.heap_cache().is_class_known(list_op) { - let type_const = ctx.trace_ctx.const_int(arg_type_addr); - ctx.trace_ctx - .record_guard(OpCode::GuardClass, &[list_op, type_const], 0); - walker_capture_snapshot_for_last_guard(ctx, op.pc)?; - } - ctx.trace_ctx - .heap_cache_mut() - .class_now_known(list_op, arg_type_addr); - if let Some(exact_w_class) = exact_w_class { - walker_guard_exact_w_class(ctx, op.pc, list_op, exact_w_class)?; - } - // `functional.py W_Range.descr_len` is already a wrapped-field - // read. Reuse that box directly; unlike the scalar length sources below, - // there is nothing to unwrap and box again. A virtual range's cached - // field makes this fold to its existing virtual wrapped-int value. - if matches!(len_source, BuiltinLenSource::RangeField) { - let boxed = crate::state::opimpl_getfield_gc_r( - ctx.trace_ctx, - list_op, - crate::descr::range_length_descr(), - ); - // Admission read the field and required it to fit a machine word - // (`w_range_length_i64`), so the trace has to pin that too: the class - // guards above prove the receiver is a range, not what its length slot - // holds. `descr_new` stores a `W_LongObject` there whenever - // `compute_range_length` leaves the machine range, and without this - // guard a later entry carrying such a range would take the recorded - // exit and hand `len()` the long straight through. - walker_guard_class( - ctx, - op.pc, - boxed, - &pyre_object::pyobject::INT_TYPE as *const _ as i64, - )?; - walker_guard_exact_w_class( - ctx, - op.pc, - boxed, - pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::INT_TYPE), - )?; - write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', boxed)?; - return Ok(Some(())); - } - // Length read. list: guard the storage strategy, then read that - // strategy's length field (rlist.py inline field for object storage; - // typed items-block length for int/float storage). str: a plain - // codepoint-length getfield (no strategy, `bh_unicodelen`). - let raw_len = match len_source { - BuiltinLenSource::ListStrategy(sid) => { - let strategy = crate::state::opimpl_getfield_gc_i( - ctx.trace_ctx, - list_op, - crate::descr::list_strategy_descr(), - ); - let sid_const = ctx.trace_ctx.const_int(sid); - ctx.trace_ctx - .record_guard(OpCode::GuardValue, &[strategy, sid_const], 0); - walker_capture_snapshot_for_last_guard(ctx, op.pc)?; - ctx.trace_ctx - .heap_cache_mut() - .replace_box(strategy, sid_const); - let len_descr = match sid { - 0 => crate::descr::list_length_descr(), - 1 => crate::descr::list_int_items_len_descr(), - _ => crate::descr::list_float_items_len_descr(), - }; - crate::state::opimpl_getfield_gc_i(ctx.trace_ctx, list_op, len_descr) - } - BuiltinLenSource::EmptyList => { - let strategy = crate::state::opimpl_getfield_gc_i( - ctx.trace_ctx, - list_op, - crate::descr::list_strategy_descr(), - ); - let empty = ctx - .trace_ctx - .const_int(pyre_object::listobject::ListStrategy::Empty as i64); - ctx.trace_ctx - .record_guard(OpCode::GuardValue, &[strategy, empty], 0); - walker_capture_snapshot_for_last_guard(ctx, op.pc)?; - ctx.trace_ctx.heap_cache_mut().replace_box(strategy, empty); - ctx.trace_ctx.const_int(0) - } - BuiltinLenSource::TupleArrayLen => { - let wrappeditems = crate::state::opimpl_getfield_gc_r( - ctx.trace_ctx, - list_op, - crate::descr::tuple_wrappeditems_descr(), - ); - crate::state::opimpl_arraylen_gc( - ctx.trace_ctx, - wrappeditems, - crate::state::pyobject_gcarray_descr(), - ) - } - BuiltinLenSource::StrField => crate::state::opimpl_getfield_gc_i( - ctx.trace_ctx, - list_op, - crate::descr::str_len_descr(), - ), - BuiltinLenSource::BytesField => crate::state::opimpl_getfield_gc_i( - ctx.trace_ctx, - list_op, - crate::descr::bytes_len_descr(), - ), - BuiltinLenSource::BytearrayField => crate::state::opimpl_getfield_gc_i( - ctx.trace_ctx, - list_op, - crate::descr::bytearray_length_descr(), - ), - BuiltinLenSource::SetField => crate::state::opimpl_getfield_gc_i( - ctx.trace_ctx, - list_op, - crate::descr::set_len_descr(), - ), - BuiltinLenSource::RangeField => unreachable!("range returned its wrapped length above"), - // `specialisedtupleobject.py length()` returns the constant - // `typelen`; there is no field to read, so the class guard above is - // the whole proof and the box below folds to a constant. - BuiltinLenSource::PairArity => ctx.trace_ctx.const_int(concrete_len as i64), - }; - ctx.trace_ctx - .set_opref_concrete(raw_len, majit_ir::Value::Int(concrete_len as i64)); - let boxed = walker_box_int(ctx, op.pc, raw_len, concrete_len as i64)?; - ctx.trace_ctx.set_opref_concrete( - boxed, - box_int_concrete(concrete_len as i64, boxed_result as i64), - ); - write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', boxed)?; - Ok(Some(())) -} - /// `isinstance(x, C)` for an ordinary class `C` — the trace shape of /// `typeobject.py` `issubtype`: /// @@ -10831,13 +9997,29 @@ pub(crate) fn try_walker_specialize_builtin_getattr( } // The name is rejected before any lookup unless it is a string, and the // resolved bytes below stay valid only while this exact string is the - // operand. A name that is not valid UTF-8 cannot match an attribute the - // fold's `&str` lookups can find, so it declines with the rest. + // operand. Keep the WTF-8 view: PyPy's RPython string carries lone + // surrogates through the same traced descriptor lookup as ASCII names. if !unsafe { pyre_object::is_exact_type(concrete_name, &pyre_object::pyobject::STR_TYPE) } { return Ok(None); } - let Ok(name) = (unsafe { pyre_object::w_str_get_wtf8(concrete_name) }).as_str() else { - return Ok(None); + let name = unsafe { pyre_object::w_str_get_wtf8(concrete_name) }; + + // Resolve the descriptor case before emitting anything. A function-valued + // class attribute is not a plain class-attribute read: getattr binds it to + // the receiver. PyPy traces that Method allocation and virtualizes it into + // the following CALL, so reproduce the same guarded allocation here. + let bound_method = unsafe { + pyre_interpreter::baseobjspace::bound_method_attr_fast_path_wtf8(concrete_obj, name) + }; + let bound_method = match bound_method { + Some((w_type, version_tag, w_descr, true)) => { + let Some(shadow) = (unsafe { walker_classify_shadow_guard(concrete_obj) }) else { + return Ok(None); + }; + Some((w_type, version_tag, w_descr, Some(shadow))) + } + Some((w_type, version_tag, w_descr, false)) => Some((w_type, version_tag, w_descr, None)), + None => None, }; let pre_emit_pos = ctx.trace_ctx.get_trace_position(); @@ -10865,6 +10047,27 @@ pub(crate) fn try_walker_specialize_builtin_getattr( )?; } + if let Some((w_type, _version_tag, w_descr, shadow)) = bound_method { + walker_emit_constant_descr_bound_method( + ctx, + op.pc, + r_args[2], + concrete_obj, + w_type, + w_descr, + shadow, + dst, + 'r', + )?; + return Ok(Some(())); + } + + let Ok(name) = name.as_str() else { + ctx.trace_ctx.cut_trace_with_snapshots(pre_emit_pos); + ctx.trace_ctx.heap_cache_mut().reset(); + return Ok(None); + }; + // Every shape the read declines has to leave the trace as it found it: the // two guards above are the premise of a fold that is no longer there, and // the residual the caller falls through to recomputes the lookup from the @@ -16326,6 +15529,22 @@ pub(crate) fn orthodox_list_append_body_and_sym( Some((sub_body, sym_ptr)) } +/// The entry frame a helper sub-walk pushes when the walk is inside an +/// inlined callee (`fbw_mode.inline_subwalk`): the callee preserved at the +/// helper's entry, so a guard in the helper rebuilds it and re-executes the +/// helper. `None` at the root level, which stays on the caller-boundary +/// resume (see `try_walker_inline_builtin_call`). An `Err` is a decline: +/// nothing has been recorded. +fn orthodox_helper_nested_entry( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, +) -> Result, InlineCallerFrameDecline> { + if !ctx.fbw_mode.inline_subwalk { + return Ok(None); + } + compute_inline_helper_call_entry_frame(ctx, op_pc).map(Some) +} + /// Enter a canonical helper body as a sub-jitcode walk from a walker fold. /// /// Publishes the call-site resume coordinate the enclosing full-body walk needs @@ -16344,59 +15563,83 @@ pub(crate) fn orthodox_list_append_body_and_sym( /// /// `fallback_label` names this site in the empty-twin coordinate note; /// `call_site_label` names it in the active-box collection. +/// +/// `nested_entry` is [`orthodox_helper_nested_entry`]'s answer for `op_pc`: +/// inside an inlined callee the helper's guards resume at that callee's +/// own coordinate (the entry frame it pushes plus the sub-walk's outer +/// coordinate/active boxes), not at the full-body sym's -- the same model +/// `try_walker_inline_builtin_call` applies. Resolving the full-body +/// coordinate from a callee `op_pc` restored the wrong frame image after +/// an overflow guard failed in `step` of `a, b = step(a, b)`. #[allow(clippy::too_many_arguments)] fn run_orthodox_helper_subwalk( ctx: &mut WalkContext<'_, '_, Sym>, op_pc: usize, sym: &Sym, sub_body: &SubJitCodeBody, + nested_entry: Option, fallback_label: &'static str, call_site_label: &'static str, int_args: &[OpRef], int_arg_concretes: &[ConcreteValue], ref_args: &[OpRef], ref_arg_concretes: &[ConcreteValue], + float_args: &[OpRef], ) -> Result<(DispatchOutcome, majit_metainterp::recorder::TracePosition), DispatchError> { - let (call_site_py_pc, vsd_value, outer_jitcode_index, call_site_marker) = unsafe { - let jc = &*sym.jitcode(); - let jc_index = jc.index as u32; - let marker = jc.payload.resume_marker_for_jitcode_pc(op_pc); - // Forward py twin first (#73 phase-3): equals the containing - // coordinate plus trivia normalization by construction; the containing - // lookup survives for the empty-twin class, and the trivia skip below - // is an identity on the twin path. - let mut py = jc - .payload - .forward_py_pc_for_jitcode_pc(op_pc) - .unwrap_or_else(|| { - crate::py_coord::note_empty_twin_fallback(fallback_label, jc.index, op_pc as i32); - crate::py_coord::containing_py_pc_for_jitcode_pc(&jc.payload.metadata, op_pc) - }); - if jc.payload.code_ptr.is_null() { - (py, sym.valuestackdepth() as i64, jc_index, marker) - } else { - let codeobj = &*jc.payload.code_ptr; - py = skip_python_trivia_forward(codeobj, py as usize) as u32; - // Read the depth off the jitcode-pc-keyed trivia twin, which equals - // `depth_at_py_pc()[skip_python_trivia_forward(containing_py_pc_for_jitcode_pc(op_pc))]` - // by construction; fall back to the py_pc-keyed static-liveness read - // where the twin is unpopulated (skeleton / fixture install). - let depth = if jc.payload.depth_trivia_populated() { - jc.payload.depth_trivia_for_jitcode_pc(op_pc) + let nested_helper = nested_entry.is_some(); + let (call_site_py_pc, vsd_value, outer_jitcode_index, call_site_marker) = if nested_helper { + ( + ctx.entry_py_pc(), + 0, + ctx.outer_jitcode_index, + ctx.outer_resume_marker_jit_pc, + ) + } else { + unsafe { + let jc = &*sym.jitcode(); + let jc_index = jc.index as u32; + let marker = jc.payload.resume_marker_for_jitcode_pc(op_pc); + // Forward py twin first (#73 phase-3): equals the containing + // coordinate plus trivia normalization by construction; the containing + // lookup survives for the empty-twin class, and the trivia skip below + // is an identity on the twin path. + let mut py = jc + .payload + .forward_py_pc_for_jitcode_pc(op_pc) + .unwrap_or_else(|| { + crate::py_coord::note_empty_twin_fallback( + fallback_label, + jc.index, + op_pc as i32, + ); + crate::py_coord::containing_py_pc_for_jitcode_pc(&jc.payload.metadata, op_pc) + }); + if jc.payload.code_ptr.is_null() { + (py, sym.valuestackdepth() as i64, jc_index, marker) } else { - crate::liveness::liveness_for(jc.payload.code_ptr) - .depth_at_py_pc() - .get(py as usize) - .copied() - }; - let vsd = match depth { - Some(d) => (sym.nlocals() + d as usize) as i64, - None => sym.valuestackdepth() as i64, - }; - (py, vsd, jc_index, marker) + let codeobj = &*jc.payload.code_ptr; + py = skip_python_trivia_forward(codeobj, py as usize) as u32; + // Read the depth off the jitcode-pc-keyed trivia twin, which equals + // `depth_at_py_pc()[skip_python_trivia_forward(containing_py_pc_for_jitcode_pc(op_pc))]` + // by construction; fall back to the py_pc-keyed static-liveness read + // where the twin is unpopulated (skeleton / fixture install). + let depth = if jc.payload.depth_trivia_populated() { + jc.payload.depth_trivia_for_jitcode_pc(op_pc) + } else { + crate::liveness::liveness_for(jc.payload.code_ptr) + .depth_at_py_pc() + .get(py as usize) + .copied() + }; + let vsd = match depth { + Some(d) => (sym.nlocals() + d as usize) as i64, + None => sym.valuestackdepth() as i64, + }; + (py, vsd, jc_index, marker) + } } }; - if sym.owns_virtualizable_shadow() { + if !nested_helper && sym.owns_virtualizable_shadow() { let li = call_site_py_pc as i64 - 1; let li_op = ctx.trace_ctx.const_int(li); crate::trace_opcode::mirror_vable_static_to_boxes( @@ -16413,28 +15656,32 @@ fn run_orthodox_helper_subwalk( Value::Int(vsd_value), ); } - let call_site_word = call_site_marker - .map(|marker| marker as i32) - .unwrap_or(majit_ir::resumedata::NO_JITCODE_PC); - let active = collect_outer_active_boxes( - sym, - ctx.trace_ctx, - ctx.registers_i, - ctx.registers_r, - ctx.registers_f, - outer_jitcode_index, - false, - call_site_word, - op_pc as i32, - OuterActiveBoxesEntryTwin::Plain, - call_site_label, - None, - &[], - // Not a branch-guard reconstruction: this is the pre-call site - // snapshot, so there is no kept operand-stack slot to report as - // unsourced. - None, - ); + let active = if nested_helper { + ctx.outer_active_boxes.clone() + } else { + let call_site_word = call_site_marker + .map(|marker| marker as i32) + .unwrap_or(majit_ir::resumedata::NO_JITCODE_PC); + collect_outer_active_boxes( + sym, + ctx.trace_ctx, + ctx.registers_i, + ctx.registers_r, + ctx.registers_f, + outer_jitcode_index, + false, + call_site_word, + op_pc as i32, + OuterActiveBoxesEntryTwin::Plain, + call_site_label, + None, + &[], + // Not a branch-guard reconstruction: this is the pre-call site + // snapshot, so there is no kept operand-stack slot to report as + // unsourced. + None, + ) + }; let saved_entry = ctx.entry_py_pc; let saved_marker = ctx.outer_resume_marker_jit_pc; @@ -16454,6 +15701,8 @@ fn run_orthodox_helper_subwalk( let walk_start = ctx.trace_ctx.get_trace_position(); let saved_fbw_mode = ctx.fbw_mode; ctx.fbw_mode.inline_subwalk = true; + let helper_frame = + nested_entry.map(|frame| InlineFrameGuard::enter(ctx.session, 0, false, vec![frame])); let walk_result = run_sub_jitcode_walk( ctx, op_pc, @@ -16462,8 +15711,9 @@ fn run_orthodox_helper_subwalk( int_arg_concretes, ref_args, ref_arg_concretes, - &[], + float_args, ); + drop(helper_frame); ctx.fbw_mode = saved_fbw_mode; ctx.entry_py_pc = saved_entry; ctx.outer_resume_marker_jit_pc = saved_marker; @@ -16476,6 +15726,48 @@ fn run_orthodox_helper_subwalk( Ok((walk_result?, walk_start)) } +/// Execute a canonical helper reached through a real codewriter +/// `inline_call_*` opcode with the same caller-boundary and descriptor-pool +/// setup as specialization-driven orthodox descent. +/// +/// Once `flatten` emits the inline-call directly, the opcode handler owns only +/// the callee body and argument lists; this wrapper recovers the full-body +/// symbol and delegates to the already-proven boundary machinery instead of +/// growing a second, subtly different helper-frame model. +pub(crate) fn run_codewriter_helper_inline_call( + ctx: &mut WalkContext<'_, '_, Sym>, + op_pc: usize, + sub_body: &SubJitCodeBody, + int_args: &[OpRef], + int_arg_concretes: &[ConcreteValue], + ref_args: &[OpRef], + ref_arg_concretes: &[ConcreteValue], + float_args: &[OpRef], +) -> Result { + let sym_ptr = ctx.fbw_mode.snapshot_sym; + if sym_ptr.is_null() || unsafe { (&*sym_ptr).jitcode().is_null() } { + return Err(DispatchError::GuardResumeCoordinateUnavailable { pc: op_pc }); + } + let sym = unsafe { &*sym_ptr }; + let nested_entry = orthodox_helper_nested_entry(ctx, op_pc) + .map_err(|_| DispatchError::GuardResumeCoordinateUnavailable { pc: op_pc })?; + run_orthodox_helper_subwalk( + ctx, + op_pc, + sym, + sub_body, + nested_entry, + "codewriter_helper_inline_commit", + "codewriter_helper_inline_call_site", + int_args, + int_arg_concretes, + ref_args, + ref_arg_concretes, + float_args, + ) + .map(|(outcome, _)| outcome) +} + /// Commit core of the #171 orthodox list-append fold, shared by the /// method-call (`try_walker_orthodox_list_append`) and LIST_APPEND-opcode /// (`try_walker_orthodox_list_append_opcode`) forms. Stamps the receiver @@ -16635,17 +15927,21 @@ pub(crate) fn orthodox_list_append_commit( // collapse to (mirror the full-body path's last_instr / valuestackdepth // publication, keyed to the append op's py_pc — the CALL for the method // form, the LIST_APPEND for the opcode form). + let nested_entry = orthodox_helper_nested_entry(ctx, op.pc) + .map_err(|_| DispatchError::callee_inline_unsupported(op.pc))?; let (walk_outcome, _walk_start) = run_orthodox_helper_subwalk( ctx, op.pc, sym, sub_body, + nested_entry, "list_append_commit", "w_list_append_call_site", &[], &[], &[self_ref, value_op], &[ConcreteValue::Ref(inner_self), ConcreteValue::Ref(value)], + &[], )?; match walk_outcome { @@ -16939,17 +16235,21 @@ pub(crate) fn orthodox_list_pop_commit( ctx.trace_ctx .set_opref_concrete(self_ref, Value::Ref(majit_ir::GcRef(inner_self as usize))); + let nested_entry = orthodox_helper_nested_entry(ctx, op.pc) + .map_err(|_| DispatchError::callee_inline_unsupported(op.pc))?; let (walk_outcome, walk_start) = run_orthodox_helper_subwalk( ctx, op.pc, sym, sub_body, + nested_entry, "list_pop_commit", "w_list_pop_end_call_site", &[], &[], &[self_ref], &[ConcreteValue::Ref(inner_self)], + &[], )?; let result = match walk_outcome { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs index f252f31dee9..9674bd431d7 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs @@ -174,6 +174,20 @@ fn void_return() -> Vec { ] } +fn int_copy_const(src: u8, dst: u8) -> Vec { + let byte = *insns_opname_to_byte() + .get("int_copy/c>i") + .expect("`int_copy/c>i` must be in insns table"); + vec![byte, src, dst] +} + +fn arraylen_gc(array: u8, descr: u16, dst: u8) -> Vec { + let byte = *insns_opname_to_byte() + .get("arraylen_gc/rd>i") + .expect("`arraylen_gc/rd>i` must be in insns table"); + vec![byte, array, descr as u8, (descr >> 8) as u8, dst] +} + /// The generated `__majit_wrap_*` gateways all put their un-lowerable call — the /// `#[dont_look_inside]` arity-error formatter — on the arm the argument-count /// check rejects into. The walk is execution-driven, so a call with the right @@ -226,6 +240,198 @@ fn a_blocker_behind_an_executed_call_is_a_decline() { assert!(!summary.body_not_walked); } +/// A constant condition is the same condition the generated walk reads. The +/// scan must follow only its executed successor; joining the dead fallthrough +/// would turn an effect and blocker that cannot execute into a false decline. +#[test] +fn a_known_goto_condition_does_not_scan_the_dead_arm() { + let symbolic = majit_translate::codewriter::call::symbolic_fnaddr_for_segments(["__len"]); + let real = 0x1234_5678i64; + + // 0: i0 = const ; 3: goto_if_not i0 -> 20 + // 7: residual(real); 13: residual(symbolic); 19: return; 20: return + // + // `int_copy/c>i` carries its source inline as one signed byte, so the + // condition is spelled by that byte and there is no `constants_i` slot to + // vary: the two bodies differ only in it. + let body = |src: u8| { + let mut code = int_copy_const(src, 0); + code.extend(goto_if_not(0, 20)); + code.extend(residual_call_with_funcbox(2)); + code.extend(residual_call_with_funcbox(3)); + code.extend(void_return()); + code.extend(void_return()); + assert_eq!(code.len(), 21); + code + }; + + let summary = + super::inline_call::summarize_body_blockers(&body(0), 1, &[0, real, symbolic], |_| None); + assert_eq!(summary.blocker_after_effect, None); + assert_eq!(summary.blocker_effect_free, None); + assert!(!summary.may_execute_effect); + + // Flip only the condition. The same body now executes the fallthrough, + // and the blocker after the real residual call must remain a decline. + let executed = + super::inline_call::summarize_body_blockers(&body(1), 1, &[0, real, symbolic], |_| None); + assert_eq!(executed.blocker_after_effect, Some(symbolic)); + assert!(executed.may_execute_effect); +} + +/// Generated builtin gateways receive an argument slice in r0 and branch on +/// its length before entering the typed body. The call site knows that length, +/// so the path-sensitive scan must not join the arity-error arm back in. +#[test] +fn a_known_wrapper_array_length_selects_only_the_executed_arity_arm() { + let symbolic = majit_translate::codewriter::call::symbolic_fnaddr_for_segments(["__len"]); + let real = 0x1234_5678i64; + + // 0: i0 = arraylen(r0); 5: goto_if_not i0 -> 22 + // 9: residual(real); 15: residual(symbolic); 21: return; 22: return + let mut code = arraylen_gc(0, 0, 0); + code.extend(goto_if_not(0, 22)); + code.extend(residual_call_with_funcbox(1)); + code.extend(residual_call_with_funcbox(2)); + code.extend(void_return()); + code.extend(void_return()); + assert_eq!(code.len(), 23); + + let scan = |len| { + super::inline_call::summarize_body_blockers_with( + &code, + 1, + &[real, symbolic], + |_, _| None, + &mut |_, _| true, + &mut |_| false, + &mut |_| None, + &[(0, len)], + ) + }; + let empty = scan(0); + assert_eq!(empty.blocker_after_effect, None); + assert!(!empty.may_execute_effect); + + let nonempty = scan(1); + assert_eq!(nonempty.blocker_after_effect, Some(symbolic)); + assert!(nonempty.may_execute_effect); +} + +/// An executed residual call the effectinfo names effect-free -- elidable, +/// loop-invariant, or `not_in_trace` -- applies nothing a rollback would have +/// to undo, so a blocker behind it stays on the rewind leg. The scan asks +/// through the `call_effect_free` hook; the helper without the hook keeps the +/// conservative reading. +#[test] +fn a_blocker_behind_an_effect_free_call_is_not_a_decline() { + let symbolic = majit_translate::codewriter::call::symbolic_fnaddr_for_segments(["__len"]); + let real = 0x1234_5678i64; + + let mut code = residual_call_with_funcbox(1); + code.extend(residual_call_with_funcbox(2)); + code.extend(void_return()); + + let mut asked = Vec::new(); + let summary = super::inline_call::summarize_body_blockers_with( + &code, + 1, + &[real, symbolic], + |_, _| None, + &mut |_, _| true, + &mut |descr_index| { + asked.push(descr_index); + true + }, + &mut |_| None, + &[], + ); + assert_eq!( + asked, + vec![0], + "only the real call's descr is asked; the blocker stops the walk" + ); + assert_eq!(summary.blocker_effect_free, Some(symbolic)); + assert_eq!(summary.blocker_after_effect, None); + assert!(!summary.may_execute_effect); + + let conservative = + super::inline_call::summarize_body_blockers(&code, 1, &[real, symbolic], |_| None); + assert_eq!(conservative.blocker_after_effect, Some(symbolic)); +} + +/// A field write into an object this body allocated is no effect: a rewind +/// drops the allocation. The same write into a register the body did not +/// fill from a `new*` keeps the conservative reading. +#[test] +fn a_write_into_a_fresh_allocation_is_not_an_effect() { + let symbolic = majit_translate::codewriter::call::symbolic_fnaddr_for_segments(["__len"]); + let table = insns_opname_to_byte(); + let new = *table + .get("new/d>r") + .expect("`new/d>r` must be in insns table"); + let setfield = *table + .get("setfield_gc_i/rid") + .expect("`setfield_gc_i/rid` must be in insns table"); + + // r1 = new; r1.field = i0; residual(symbolic) + let mut code = vec![new, 0, 0, 1, setfield, 1, 0, 0, 0]; + code.extend(residual_call_with_funcbox(1)); + code.extend(void_return()); + let summary = super::inline_call::summarize_body_blockers(&code, 1, &[symbolic], |_| None); + assert_eq!(summary.blocker_effect_free, Some(symbolic)); + assert_eq!(summary.blocker_after_effect, None); + assert!(!summary.may_execute_effect); + + // r2 was never allocated here: the write is an effect. + let mut code = vec![new, 0, 0, 1, setfield, 2, 0, 0, 0]; + code.extend(residual_call_with_funcbox(1)); + code.extend(void_return()); + let summary = super::inline_call::summarize_body_blockers(&code, 1, &[symbolic], |_| None); + assert_eq!(summary.blocker_after_effect, Some(symbolic)); + assert!(summary.may_execute_effect); +} + +/// A `switch` whose arm table is known continues only at its arms and the +/// fallthrough, so the state at one arm's switch does not reach a region no +/// arm names; without the table every instruction start is a successor. +#[test] +fn a_switch_with_a_known_table_does_not_leak_its_state_to_every_start() { + let symbolic = majit_translate::codewriter::call::symbolic_fnaddr_for_segments(["__len"]); + let real = 0x1234_5678i64; + let switch = *insns_opname_to_byte() + .get("switch/id") + .expect("`switch/id` must be in insns table"); + + // 0: residual(real) 6: switch i0 d0 10: return 11: residual(symbolic) 17: return + let mut code = residual_call_with_funcbox(1); + code.extend([switch, 0, 0, 0]); + code.extend(void_return()); + code.extend(residual_call_with_funcbox(2)); + code.extend(void_return()); + assert_eq!(code.len(), 18); + + let known = super::inline_call::summarize_body_blockers_with( + &code, + 1, + &[real, symbolic], + |_, _| None, + &mut |_, _| true, + &mut |_| false, + &mut |_| Some(vec![10]), + &[], + ); + assert_eq!( + known.blocker_after_effect, None, + "pc 11 is no arm and no fallthrough" + ); + assert_eq!(known.blocker_effect_free, None); + + let widened = + super::inline_call::summarize_body_blockers(&code, 1, &[real, symbolic], |_| None); + assert_eq!(widened.blocker_after_effect, Some(symbolic)); +} + /// A callee's blocker is judged by what has run in the caller, not by what had /// run in the callee: the same body reached with an effect behind it holds a /// blocker no rollback covers. diff --git a/pyre/pyre-jit-trace/src/jitcode_runtime.rs b/pyre/pyre-jit-trace/src/jitcode_runtime.rs index e6a82a6c711..78a29092dd0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_runtime.rs +++ b/pyre/pyre-jit-trace/src/jitcode_runtime.rs @@ -203,6 +203,41 @@ pub fn get_jitcode_by_index(index: usize) -> Option> { get_jitcode_ref_by_index(index).cloned() } +/// Runtime wrappers for canonical source-translator JitCodes embedded as +/// `AbstractDescr` operands in per-Python-function bytecode. RPython owns +/// these in the process-wide `CallControl.jitcodes[graph]` map, so this cache +/// is process-wide too rather than another copy in each thread. The slots are +/// index-aligned with the canonical JitCode table. +static RUNTIME_JITCODE_CELLS: std::sync::OnceLock< + Box<[std::sync::OnceLock>]>, +> = std::sync::OnceLock::new(); + +fn runtime_jitcode_cells() -> &'static [std::sync::OnceLock>] +{ + RUNTIME_JITCODE_CELLS.get_or_init(|| { + (0..jitcode_count()) + .map(|_| std::sync::OnceLock::new()) + .collect::>() + .into_boxed_slice() + }) +} + +pub fn get_runtime_jitcode_by_index( + index: usize, +) -> Option> { + let cell = runtime_jitcode_cells().get(index)?; + Some( + cell.get_or_init(|| { + let canonical = get_jitcode_by_index(index) + .expect("runtime JitCode cell index must resolve canonically"); + Arc::new(majit_metainterp::jitcode::JitCode::from_canonical( + (*canonical).clone(), + )) + }) + .clone(), + ) +} + /// The source translator's exact `Assembler.indirectcalltargets` set as /// references into `all_jitcodes`. /// @@ -399,16 +434,6 @@ static LIST_POP_END_JITCODE_INDEX: OnceLock> = OnceLock::new(); /// Cached `ALL_JITCODES` index of `w_tuple_getitem`, resolved by graph key /// rather than by name -- see [`compute_pathed_jitcode_index`]. static TUPLE_GETITEM_JITCODE_INDEX: OnceLock> = OnceLock::new(); -/// Cached `ALL_JITCODES` index of `invert_inner`, resolved by graph key: `neg` -/// and `invert` held indices 2798/2809 in only two of eight observed cache -/// generations, so an index is not stable across builds and a path is. -static INVERT_INNER_JITCODE_INDEX: OnceLock> = OnceLock::new(); -/// Cached `ALL_JITCODES` index of `neg_inner`, resolved by graph key for the -/// reason given above. -static NEG_INNER_JITCODE_INDEX: OnceLock> = OnceLock::new(); -/// Cached `ALL_JITCODES` index of `pos_inner`, resolved by graph key for the -/// reason given above. -static POS_INNER_JITCODE_INDEX: OnceLock> = OnceLock::new(); /// Cached `ALL_JITCODES` index of the interpreter-source /// `load_super_attr_value_w` body. The fused opcode reaches this ordinary /// graph so `_super_check` and `W_Super.getattribute` are traced from their @@ -472,6 +497,49 @@ pub(crate) fn pathed_jitcode(canonical_path: &str) -> Option> { get_jitcode_by_index(compute_pathed_jitcode_index(canonical_path)?) } +thread_local! { + /// `canonical_path` → its `ALL_JITCODES` index, resolved once per thread. + /// The `None` answer is cached too: a helper absent from the build-time + /// pipeline is asked for on every consult of its site. + /// + /// This is deliberately only a disposable lookup cache, not the owner of + /// RPython's process-wide `CallControl.jitcodes`: `jitcode_index().paths` + /// is frozen build data, and the cache retains only an index or `None` — + /// never a JitCode identity, GC reference, or semantic state. Dropping or + /// duplicating it on another thread can change lookup cost only. + static PATHED_JITCODE_INDEX: std::cell::RefCell>> = + std::cell::RefCell::new(std::collections::HashMap::new()); +} + +/// [`pathed_jitcode`] for a descent's fixed target, cached per thread. The +/// descents this serves are rows of a table (`specialize.rs UnaryDescent`), +/// so the path is a literal and the cache is keyed by it. +pub(crate) fn pathed_jitcode_cached(canonical_path: &'static str) -> Option> { + let idx = PATHED_JITCODE_INDEX.with(|cache| { + *cache + .borrow_mut() + .entry(canonical_path) + .or_insert_with(|| compute_pathed_jitcode_index(canonical_path)) + })?; + get_jitcode_by_index(idx) +} + +/// Runtime-wrapper sibling of [`pathed_jitcode_cached`], for a codewriter +/// `inline_call_*` operand. RPython's JitCode is both the body and its +/// `AbstractDescr`; pyre keeps the serializable canonical body separate from +/// the runtime descr pool, so this is the single identity-preserving join. +pub fn pathed_runtime_jitcode_cached( + canonical_path: &'static str, +) -> Option> { + let idx = PATHED_JITCODE_INDEX.with(|cache| { + *cache + .borrow_mut() + .entry(canonical_path) + .or_insert_with(|| compute_pathed_jitcode_index(canonical_path)) + })?; + get_runtime_jitcode_by_index(idx) +} + /// Resolve an ordinary portal-closure JitCode by its unique graph leaf name. /// Prefer stable graph paths at build time; this runtime helper exists for /// diagnostics and tests whose serialized artifact stores names only. @@ -501,6 +569,15 @@ pub fn list_pop_end_jitcode() -> Option> { get_jitcode_by_index(idx) } +/// The wrapped-name interpreter-source value half of `LOAD_SUPER_ATTR`, resolved by the +/// graph key the codewriter allocated it under and cached process-wide. +pub fn load_super_attr_value_jitcode() -> Option> { + let idx = (*LOAD_SUPER_ATTR_VALUE_JITCODE_INDEX.get_or_init(|| { + compute_pathed_jitcode_index("pyre_interpreter::eval::load_super_attr_value_w") + }))?; + get_jitcode_by_index(idx) +} + /// The charon `w_tuple_getitem` body in `ALL_JITCODES`, resolved by the graph /// key the codewriter allocated it under and cached process-wide. `None` if the /// helper is absent from the build-time pipeline. @@ -517,71 +594,6 @@ pub fn list_pop_end_jitcode() -> Option> { /// `_known` reader is what keeps the length test in the trace: the caller's /// trace-time range check proves only that THIS receiver is long enough, while /// the recorded `arraylen` guard is what holds for the next one. -/// The charon `invert_inner` body in `ALL_JITCODES`, resolved by the graph key -/// the codewriter allocated it under and cached process-wide. `None` if the -/// helper is absent from the build-time pipeline. -/// -/// This is `descroperation.rs` `invert` past its `__invert__` override probe -/// and its bool slot -- the two arms a descent cannot take. What is left is the -/// `int` arm, the `long` arm, the instance fallback and the terminal -/// `TypeError`, so a caller that has pinned an exact `int` or `long` receiver -/// records one of the first two and nothing else. -pub fn invert_inner_jitcode() -> Option> { - let idx = (*INVERT_INNER_JITCODE_INDEX.get_or_init(|| { - compute_pathed_jitcode_index("pyre_interpreter::objspace::descroperation::invert_inner") - }))?; - get_jitcode_by_index(idx) -} - -/// The charon `neg_inner` body in `ALL_JITCODES`, resolved by the graph key the -/// codewriter allocated it under and cached process-wide. `None` if the helper is -/// absent from the build-time pipeline. -/// -/// This is `descroperation.rs` `neg` past its `__neg__` override probe -- the -/// one arm a descent cannot take. What is left is the integer arm (including -/// the `checked_neg` promotion of `INT_MIN`), the `long`, `float` and `complex` -/// arms, the instance fallback and the terminal `TypeError`, so a caller that -/// has pinned an exact `int` or `long` receiver records one of the first two -/// and nothing else. -pub fn neg_inner_jitcode() -> Option> { - let idx = (*NEG_INNER_JITCODE_INDEX.get_or_init(|| { - compute_pathed_jitcode_index("pyre_interpreter::objspace::descroperation::neg_inner") - }))?; - get_jitcode_by_index(idx) -} - -/// The charon `pos_inner` body in `ALL_JITCODES`, resolved by the graph key the -/// codewriter allocated it under and cached process-wide. `None` if the helper is -/// absent from the build-time pipeline. -/// -/// This is `descroperation.rs` `pos` past its `__pos__` override probe -- the -/// one arm a descent cannot take. What is left is the exact-int identity, the -/// bool/int rewrap, the long/float/complex arms, the instance fallback and the -/// terminal `TypeError`, so a caller that has pinned an exact `int` or `long` -/// receiver records one of the identity arms and nothing else. -/// -/// `+x` is `CALL_INTRINSIC_1`, so the only route to this body runs through -/// `OpcodeStepExecutor::unary_positive`, which is declared without a body for -/// that reason: a default there is the graph the walker takes, and `PyFrame`'s -/// implementation -- and with it `pos` and `pos_inner` -- is never reached. -/// `None` here is the helper's absence from the build-time pipeline, and the -/// identity fold behind the descent still serves exact-int `+x`. -pub fn pos_inner_jitcode() -> Option> { - let idx = (*POS_INNER_JITCODE_INDEX.get_or_init(|| { - compute_pathed_jitcode_index("pyre_interpreter::objspace::descroperation::pos_inner") - }))?; - get_jitcode_by_index(idx) -} - -/// The wrapped-name interpreter-source value half of `LOAD_SUPER_ATTR`, resolved by the -/// graph key the codewriter allocated it under and cached process-wide. -pub fn load_super_attr_value_jitcode() -> Option> { - let idx = (*LOAD_SUPER_ATTR_VALUE_JITCODE_INDEX.get_or_init(|| { - compute_pathed_jitcode_index("pyre_interpreter::eval::load_super_attr_value_w") - }))?; - get_jitcode_by_index(idx) -} - pub fn tuple_getitem_jitcode() -> Option> { let idx = (*TUPLE_GETITEM_JITCODE_INDEX.get_or_init(|| { compute_pathed_jitcode_index("pyre_object::tupleobject::w_tuple_getitem") @@ -3115,6 +3127,40 @@ mod tests { assert!(!jitcodes.is_empty(), "expected at least one jitcode"); } + #[test] + fn canonical_runtime_wrapper_identity_is_shared_across_threads() { + let first = get_runtime_jitcode_by_index(0).expect("first canonical runtime JitCode"); + let second = std::thread::spawn(|| { + get_runtime_jitcode_by_index(0).expect("first canonical runtime JitCode on child") + }) + .join() + .unwrap(); + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn pathed_lookup_cache_is_disposable_without_changing_identity() { + let path: &'static str = jitcode_index() + .paths + .iter() + .find(|path| !path.is_empty()) + .expect("at least one graph-backed JitCode") + .as_str(); + let first = pathed_runtime_jitcode_cached(path).expect("canonical path"); + PATHED_JITCODE_INDEX.with(|cache| cache.borrow_mut().clear()); + let again = pathed_runtime_jitcode_cached(path).expect("uncached canonical path"); + let other = std::thread::spawn(move || { + pathed_runtime_jitcode_cached(path).expect("canonical path on another thread") + }) + .join() + .unwrap(); + assert!(Arc::ptr_eq(&first, &again)); + assert!(Arc::ptr_eq(&first, &other)); + assert!(pathed_runtime_jitcode_cached("__missing_jitcode_path__").is_none()); + PATHED_JITCODE_INDEX.with(|cache| cache.borrow_mut().clear()); + assert!(pathed_runtime_jitcode_cached("__missing_jitcode_path__").is_none()); + } + #[test] fn indirect_target_lookup_decodes_only_the_matched_jitcode() { let (&fnaddr, &index) = INDIRECTCALLTARGET_BY_FNADDR @@ -3624,6 +3670,7 @@ mod tests { "getlistitem_gc_f/ridd>f", "getlistitem_gc_i/ridd>i", "getlistitem_gc_r/ridd>r", + "guard_class/r>i", "int_between/iii>i", "newlist/idddd>r", "newlist_clear/idddd>r", diff --git a/pyre/pyre-jit-trace/src/runtime_fnaddr_patch.rs b/pyre/pyre-jit-trace/src/runtime_fnaddr_patch.rs index 37a3bf6d426..d890064a210 100644 --- a/pyre/pyre-jit-trace/src/runtime_fnaddr_patch.rs +++ b/pyre/pyre-jit-trace/src/runtime_fnaddr_patch.rs @@ -95,6 +95,15 @@ pub fn patch_constants_i_fnaddrs(jitcodes: &mut [Arc]) { } } +/// The runtime address published for `path` in `jit_trace_fnaddrs`, or +/// `None` when the path is not published. A walker fold that recognises a +/// residual by its callee compares the call's funcbox against this. +pub fn runtime_fnaddr_by_path(path: &str) -> Option { + static RUNTIME_FNADDRS: LazyLock> = + LazyLock::new(|| pyre_interpreter::jit_trace_fnaddrs().into_iter().collect()); + RUNTIME_FNADDRS.get(path).copied() +} + static FNADDR_CORRESPONDENCE: LazyLock> = LazyLock::new(|| { let build_bindings = build_time_fnaddr_bindings(); let runtime_bindings = pyre_interpreter::jit_trace_fnaddrs(); diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index d1df8c60f35..2d2cf55dc6e 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -10038,6 +10038,24 @@ pub(crate) fn portal_activation_result(frame: &mut PyFrame) -> PyResult { enter_portal(frame, bracketed_without_resume) } +/// `execute_frame`'s activation entry for an interpreter/blackhole portal +/// door: the logical half of the stack check `pyframe.py` marks with +/// `insert_stack_check_here`, and then the activation unit that check bounds. +/// +/// The ordinary door runs the same pair in [`eval_with_jit`]. A portal door +/// reached by `bhimpl_recursive_call_*` does not pass through it. On a guard +/// exit, the compiled activation seam charged the depth ahead of the failing +/// guard and deliberately did not give it back, so the depth read here is the +/// one that charge produced. On an uncompiled recursive portal call, this +/// check/charge order is the same as [`eval_with_jit`]. The backend's fragment +/// prologue already owns the independent native-stack check. +fn enter_portal_activation( + frame: &mut PyFrame, +) -> Result { + pyre_interpreter::stack_check::check_recursion_depth()?; + Ok(pyre_interpreter::call::enter_recursive_frame(frame)) +} + /// Activation whose execution-context vref bracket is emitted by compiled /// code around CALL_ASSEMBLER. /// @@ -10047,6 +10065,9 @@ pub(crate) fn portal_activation_result(frame: &mut PyFrame) -> PyResult { /// `execute_frame`'s hook bracket. The compiled continuation records the /// matching chain restore after the call returns. pub(crate) fn portal_traced_activation_result(frame: &mut PyFrame) -> PyResult { + // `record_activation_charge` already emitted this door's logical depth + // check into the compiled trace. Only a guard exit that re-enters through + // `portal_activation_result` needs to repeat the check in Rust. let _recursion_depth = pyre_interpreter::call::enter_recursive_frame(frame); let mut frame_root = FrameRoot::new(frame); frame_root.frame().fix_array_ptrs(); @@ -10069,7 +10090,7 @@ pub(crate) fn portal_traced_activation_result(frame: &mut PyFrame) -> PyResult { /// Account before constructing `FrameRoot`, because a moving GC may change the /// frame's address while the activation remains the same. fn enter_portal(frame: &mut PyFrame, body: fn(&mut FrameRoot) -> PyResult) -> PyResult { - let _recursion_depth = pyre_interpreter::call::enter_recursive_frame(frame); + let _recursion_depth = enter_portal_activation(frame)?; let mut frame_root = FrameRoot::new(frame); frame_root.frame().fix_array_ptrs(); let _frame_guard = pyre_interpreter::eval::install_current_frame(frame_root.frame()); diff --git a/pyre/pyre-jit/src/jit/assembler.rs b/pyre/pyre-jit/src/jit/assembler.rs index e2eff8053a3..1c490f28cbd 100644 --- a/pyre/pyre-jit/src/jit/assembler.rs +++ b/pyre/pyre-jit/src/jit/assembler.rs @@ -1678,6 +1678,13 @@ fn dispatch_op( opname if opname.starts_with("residual_call_") => { dispatch_residual_call(state, opname, args, result); } + // `jtransform.py handle_regular_call` shape: leading JitCode descr, + // then the non-empty kind lists selected by `rewrite_call`, and an + // optional typed result. Unlike the older pyre-only nested-call + // adapter, this emits the canonical dR/dIR/dIRF byte layout. + opname if opname.starts_with("inline_call_") => { + dispatch_inline_call(state, opname, args, result); + } other => panic!( "assemble(): unimplemented opname {:?} — add a builder mapping in jit/assembler.rs", other @@ -1685,6 +1692,103 @@ fn dispatch_op( } } +fn dispatch_inline_call( + state: &mut AssemblyState, + opname: &str, + args: &[Operand], + result: Option<&Register>, +) { + let tail = &opname["inline_call_".len()..]; + let (kinds, reskind) = tail + .rsplit_once('_') + .unwrap_or_else(|| panic!("malformed inline_call opname: {opname:?}")); + assert!(matches!(kinds, "r" | "ir" | "irf")); + assert!(matches!(reskind, "i" | "r" | "f" | "v")); + + let target = match args.first() { + Some(Operand::Descr(descr)) => match &**descr { + DescrOperand::JitCode(jitcode) => Arc::clone(jitcode), + other => panic!("inline_call expects a JitCode descr, got {other:?}"), + }, + other => panic!("inline_call expects a leading Descr operand, got {other:?}"), + }; + let target_idx = state.builder.add_sub_jitcode_arc(target); + + let mut cursor = 1usize; + let mut args_i = None; + let mut args_r = None; + let mut args_f = None; + if kinds.contains('i') { + args_i = Some( + expect_list_regs_or_pool(state, &args[cursor], Kind::Int) + .into_iter() + .map(u16::from) + .collect::>(), + ); + cursor += 1; + } + if kinds.contains('r') { + args_r = Some( + expect_list_regs_or_pool(state, &args[cursor], Kind::Ref) + .into_iter() + .map(u16::from) + .collect::>(), + ); + cursor += 1; + } + if kinds.contains('f') { + args_f = Some( + expect_list_regs_or_pool(state, &args[cursor], Kind::Float) + .into_iter() + .map(u16::from) + .collect::>(), + ); + cursor += 1; + } + assert_eq!(cursor, args.len(), "inline_call has trailing operands"); + + let result = match reskind { + "v" => { + assert!(result.is_none(), "void inline_call cannot have a result"); + None + } + "i" => Some(( + majit_metainterp::jitcode::JitArgKind::Int, + expect_result_reg(result, Kind::Int, "inline_call_i needs result"), + )), + "r" => Some(( + majit_metainterp::jitcode::JitArgKind::Ref, + expect_result_reg(result, Kind::Ref, "inline_call_r needs result"), + )), + "f" => Some(( + majit_metainterp::jitcode::JitArgKind::Float, + expect_result_reg(result, Kind::Float, "inline_call_f needs result"), + )), + _ => unreachable!(), + }; + let key: &'static str = match (kinds, reskind) { + ("r", "i") => "inline_call_r_i/dR>i", + ("r", "r") => "inline_call_r_r/dR>r", + ("r", "v") => "inline_call_r_v/dR", + ("ir", "i") => "inline_call_ir_i/dIR>i", + ("ir", "r") => "inline_call_ir_r/dIR>r", + ("ir", "v") => "inline_call_ir_v/dIR", + ("irf", "i") => "inline_call_irf_i/dIRF>i", + ("irf", "r") => "inline_call_irf_r/dIRF>r", + ("irf", "f") => "inline_call_irf_f/dIRF>f", + ("irf", "v") => "inline_call_irf_v/dIRF", + _ => panic!("unsupported canonical inline_call shape: {opname}"), + }; + state.builder.canonical_inline_call( + key, + target_idx, + args_i.as_deref(), + args_r.as_deref(), + args_f.as_deref(), + result, + ); +} + /// Consumer for the `residual_call_{kinds}_{reskind}` shape /// emitted by `flatten`'s `build_*_residual_call_*_insn` family. /// @@ -3239,6 +3343,50 @@ mod tests { assert!(descr.extra_info.is_call_release_gil()); } + #[test] + fn assemble_inline_call_r_r_emits_the_canonical_descr_varlist_shape() { + let target = Arc::new(JitCodeBuilder::default().finish()); + let mut ssarepr = SSARepr::new("inline_call_r_r"); + ssarepr.insns.push(Insn::op_with_result( + "inline_call_r_r", + vec![ + Operand::descr(DescrOperand::JitCode(Arc::clone(&target))), + Operand::ListOfKind(ListOfKind::new( + Kind::Ref, + vec![Operand::Register(Register::new(Kind::Ref, 0))], + )), + ], + Register::new(Kind::Ref, 1), + )); + + let jitcode = assemble( + &mut ssarepr, + JitCodeBuilder::default(), + Some(NumRegs { + ref_: 2, + ..NumRegs::default() + }), + ); + + assert_eq!( + jitcode.code, + vec![majit_translate::insns::BC_INLINE_CALL_R_R, 0, 0, 1, 0, 1,] + ); + match &jitcode.exec.descrs[0] { + majit_metainterp::jitcode::RuntimeBhDescr::JitCode(stored) => { + assert!(Arc::ptr_eq(stored, &target)); + } + other => panic!("inline_call descr slot is not a JitCode: {other:?}"), + } + assert_eq!( + jitcode + .resulttypes + .as_ref() + .and_then(|resulttypes| resulttypes.get(&jitcode.code.len())), + Some(&'r') + ); + } + #[test] fn residual_call_r_v_preserves_word_returning_and_genuine_void_abis() { let mut word_builder = JitCodeBuilder::default(); diff --git a/pyre/pyre-jit/src/jit/flatten.rs b/pyre/pyre-jit/src/jit/flatten.rs index 2a21683aa80..db0b9616a11 100644 --- a/pyre/pyre-jit/src/jit/flatten.rs +++ b/pyre/pyre-jit/src/jit/flatten.rs @@ -17,6 +17,7 @@ //! that appears inside a tuple. use std::rc::Rc; +use std::sync::Arc; use majit_ir::Descr; use majit_translate::codewriter::flatten::reorder_renaming_list; @@ -403,6 +404,12 @@ impl SwitchDictDescr { /// `_labels` and the assembler sees a finalised runtime descr. #[derive(Debug, Clone)] pub enum DescrOperand { + /// `call.py CallControl.get_jitcode`'s `JitCode`, which is itself an + /// `AbstractDescr` in RPython and is therefore the leading `d` operand of + /// every `inline_call_*` emitted by `jtransform.py + /// handle_regular_call`. The runtime wrapper owns the per-body descr + /// pool needed when a Python-code JitCode embeds a translated helper. + JitCode(Arc), /// Runtime descr already materialised as `BhDescr`. Bh(BhDescr), /// SSARepr-side `SwitchDictDescr` before `attach()`; liveness reads @@ -3650,26 +3657,23 @@ pub struct LoweringContext { /// codewriter pushes back onto the stack. `jit_set_function_attribute` /// stamps a typed field (`Plain` — runs no user code, never raises). pub set_function_attribute_fn_idx: u16, - /// `unary_negative_fn` descrs-pool index. UNARY_NEGATIVE records the - /// flowspace `neg(value)` op (operation.py) lowered to - /// `residual_call_r_r(ConstInt(fn_idx), ListR([value]), Descr) → reg` via - /// [`lower_unary_negative_hlop_to_insn`] (the single-Ref FORMAT_SIMPLE - /// shape); `bh_unary_negative_fn` computes `-value` (a user `__neg__` - /// may force virtualizables → `MayForce`). + /// `unary_negative_fn` descrs-pool index. It is the fallback used only + /// when the source-translator body for flowspace `neg(value)` is absent; + /// [`lower_unary_negative_hlop_to_insn`] normally emits the canonical + /// `inline_call_r_r` to that body. The fallback may invoke user `__neg__` + /// and therefore remains `MayForce`. pub unary_negative_fn_idx: u16, - /// `unary_invert_fn` descrs-pool index. UNARY_INVERT records the - /// object-space `invert(value)` op (pyopcode.py:653) lowered to - /// `residual_call_r_r(ConstInt(fn_idx), ListR([value]), Descr) → reg` via - /// [`lower_unary_invert_hlop_to_insn`] (the single-Ref FORMAT_SIMPLE - /// shape); `bh_unary_invert_fn` computes `~value` (a user `__invert__` - /// may force virtualizables → `MayForce`). + /// `unary_invert_fn` descrs-pool index. It is the fallback used only when + /// the source-translator body for object-space `invert(value)` is absent; + /// [`lower_unary_invert_hlop_to_insn`] normally emits the canonical + /// `inline_call_r_r` to that body. The fallback may invoke user + /// `__invert__` and therefore remains `MayForce`. pub unary_invert_fn_idx: u16, - /// `unary_positive_fn` descrs-pool index. UNARY_POSITIVE records the - /// object-space `pos(value)` op (pyopcode.py:649) lowered to - /// `residual_call_r_r(ConstInt(fn_idx), ListR([value]), Descr) → reg` via - /// [`lower_unary_positive_hlop_to_insn`] (the single-Ref FORMAT_SIMPLE - /// shape); `bh_unary_positive_fn` computes `+value` (a user `__pos__` - /// may force virtualizables → `MayForce`). + /// `unary_positive_fn` descrs-pool index. It is the fallback used only + /// when the source-translator body for object-space `pos(value)` is absent; + /// [`lower_unary_positive_hlop_to_insn`] normally emits the canonical + /// `inline_call_r_r` to that body. The fallback may invoke user `__pos__` + /// and therefore remains `MayForce`. pub unary_positive_fn_idx: u16, /// `load_common_constant_fn` descrs-pool index. LOAD_COMMON_CONSTANT /// records the `load_common_constant(disc)` HLOp lowered to @@ -6443,11 +6447,77 @@ where )) } +/// The body of a fixed callee path whose host addresses this build has fully +/// bound — what an `inline_call_*` may name — or `None` for the residual-call +/// case. +/// +/// Three conditions answer `None`. The path may resolve to no JitCode at all, +/// in a build whose translation pipeline never assembled that graph. The +/// resolved body may carry no `startpoints`, the record of which byte offsets +/// begin an instruction; without it the scan below cannot tell an opcode byte +/// from an operand byte of equal value, and so inspects nothing. Or the body +/// may still hold a `symbolic_fnaddr_for_path` placeholder — transitively, +/// through the `inline_call_*` callees it names — standing for a call this +/// build never bound to a host address. +/// +/// That last condition is pyre's own admission rule and has no upstream +/// counterpart: after translation every graph has a real `getfunctionptr` +/// address, so an unbound placeholder cannot exist there. Upstream's nearest +/// gate is about something else and is not transitive — `call.py +/// CallControl.is_candidate` is `graph in self.candidate_graphs`, and +/// `policy.py JitPolicy.look_inside_graph`, which seeds that set, reads one +/// graph's own operations, its own back-edges and its own +/// `_jit_look_inside_` / `_elidable_function_` flags. +/// +/// What is upstream is that the decision belongs here, before emission: +/// `jtransform.py handle_regular_call` emits `inline_call_*` only for a graph +/// `graphs_from` returned, and `pyjitpl.py _opimpl_inline_call1`/`2`/`3` +/// accordingly have no decline path +/// — nor does pyre's walker route for the op (`specialize.rs +/// run_codewriter_helper_inline_call`). A walk that meets an unbound symbolic +/// target aborts the whole trace, not just the callee, whereas the residual +/// fallback the caller keeps executes it. +/// +/// The transitive scan is `JitCode::reachable_symbolic_residuals`, memoized on +/// the JitCode, so a body walk costs O(body) once per callee rather than once +/// per lowered site. It follows each `inline_call_*` through the shared +/// build-time descr pool, which a body assembled at build time does not carry +/// per-jitcode; the pool is installed first because a scan that resolves no +/// callee also reports no target, which would read here as a clean body. +fn fully_bound_callee_body( + canonical_path: &'static str, +) -> Option> { + pyre_jit_trace::jitcode_runtime::install_global_build_descr_pool(); + let jitcode = pyre_jit_trace::jitcode_runtime::pathed_runtime_jitcode_cached(canonical_path)?; + jitcode.startpoints.as_ref()?; + if !jitcode.reachable_symbolic_residuals().targets.is_empty() { + return None; + } + Some(jitcode) +} + +fn build_orthodox_inline_call_r_r( + canonical_path: &'static str, + value: Operand, + dst_reg: Register, +) -> Option { + let jitcode = fully_bound_callee_body(canonical_path)?; + Some(Insn::op_with_result( + "inline_call_r_r", + vec![ + Operand::descr(DescrOperand::JitCode(jitcode)), + Operand::ListOfKind(ListOfKind::new(Kind::Ref, vec![value])), + ], + dst_reg, + )) +} + /// Lower the UNARY_NEGATIVE flowspace op `neg(value)` → `result: Ref` -/// (operation.py:466) to `residual_call_r_r(ConstInt(unary_negative_fn_idx), -/// ListR([value]), Descr) → reg`, the single-Ref -/// [`lower_format_simple_hlop_to_insn`] shape. `bh_unary_negative_fn` -/// computes `-value`; a user `__neg__` may force virtualizables → `MayForce`. +/// (operation.py `neg`) to the canonical +/// `inline_call_r_r(JitCode, ListR([value])) → reg` emitted by RPython's +/// `jtransform.py handle_regular_call`. A build whose `descroperation::neg` +/// body `fully_bound_callee_body` declines retains the MayForce residual +/// fallback. /// /// Returns `None` for non-`neg` opnames so the caller can fall through to /// other lowering arms. @@ -6469,6 +6539,13 @@ where Some(super::flow::FlowValue::Variable(var)) => get_register(*var), _ => return None, }; + if let Some(insn) = build_orthodox_inline_call_r_r( + "pyre_interpreter::objspace::descroperation::neg", + value.clone(), + dst_reg, + ) { + return Some(insn); + } Some(build_residual_call_r_r_insn_from_operands( ctx.unary_negative_fn_idx, vec![value], @@ -6480,10 +6557,12 @@ where /// Lower the UNARY_INVERT object-space op `invert(value)` → `result: Ref` /// (pyopcode.py `unaryoperation("invert")`) to -/// `residual_call_r_r(ConstInt(unary_invert_fn_idx), ListR([value]), -/// Descr) → reg`, the single-Ref [`lower_format_simple_hlop_to_insn`] shape. -/// `bh_unary_invert_fn` computes `~value`; a user `__invert__` may force -/// virtualizables → `MayForce`. +/// the canonical `inline_call_r_r(JitCode, ListR([value])) → reg` emitted by +/// RPython's `jtransform.py handle_regular_call`. This lets the ordinary +/// inline-call dispatcher trace `descroperation::invert` rather than asking a +/// residual-call specialization to rediscover it. A build whose body +/// `fully_bound_callee_body` declines retains the `bh_unary_invert_fn` +/// MayForce fallback. /// /// Returns `None` for non-`invert` opnames so the caller can fall /// through to other lowering arms. @@ -6505,6 +6584,13 @@ where Some(super::flow::FlowValue::Variable(var)) => get_register(*var), _ => return None, }; + if let Some(insn) = build_orthodox_inline_call_r_r( + "pyre_interpreter::objspace::descroperation::invert", + value.clone(), + dst_reg, + ) { + return Some(insn); + } Some(build_residual_call_r_r_insn_from_operands( ctx.unary_invert_fn_idx, vec![value], @@ -6516,10 +6602,10 @@ where /// Lower the UNARY_POSITIVE object-space op `pos(value)` → `result: Ref` /// (pyopcode.py `unaryoperation("pos")`) to -/// `residual_call_r_r(ConstInt(unary_positive_fn_idx), ListR([value]), -/// Descr) → reg`, the single-Ref [`lower_format_simple_hlop_to_insn`] shape. -/// `bh_unary_positive_fn` computes `+value`; a user `__pos__` may force -/// virtualizables → `MayForce`. +/// the canonical `inline_call_r_r(JitCode, ListR([value])) → reg` emitted by +/// RPython's `jtransform.py handle_regular_call`. A build whose +/// `descroperation::pos` body `fully_bound_callee_body` declines retains the +/// MayForce residual fallback. /// /// Returns `None` for non-`pos` opnames so the caller can fall through /// to other lowering arms. @@ -6541,6 +6627,13 @@ where Some(super::flow::FlowValue::Variable(var)) => get_register(*var), _ => return None, }; + if let Some(insn) = build_orthodox_inline_call_r_r( + "pyre_interpreter::objspace::descroperation::pos", + value.clone(), + dst_reg, + ) { + return Some(insn); + } Some(build_residual_call_r_r_insn_from_operands( ctx.unary_positive_fn_idx, vec![value], @@ -13317,20 +13410,19 @@ mod tests { } } - /// Shared body for the single-Ref unary HLOp lowerings (`invert`, `pos`, - /// `not_`): each records `(value)` and lowers to - /// `residual_call_r_r(ConstInt(expected_fn_idx), ListR([value]), Descr)` - /// returning reg. `lower` selects the specific `lower_unary_*_hlop_to_insn`. - fn assert_unary_lowering_emits_residual( + /// Lower the shared single-Ref unary HLOp fixture — `value` in Ref + /// register 101, result in Ref register 102 — and return the emitted + /// `Insn::Op` parts. Every unary lowering below is handed the same + /// fixture, so the two expectations differ only in `opname` and `args[0]`. + fn lower_unary_hlop_fixture( op_name: &str, - expected_fn_idx: i64, lower: impl FnOnce( &super::super::flow::SpaceOperation, &LoweringContext, &mut dyn FnMut(Variable) -> Register, &mut dyn FnMut(&Constant) -> Operand, ) -> Option, - ) { + ) -> (String, Vec, Option) { let value_var = Variable::new(VariableId(8), Kind::Ref); let result_var = Variable::new(VariableId(9), Kind::Ref); let (ctx, _, _) = load_attr_lowering_fixture(); @@ -13359,50 +13451,139 @@ mod tests { opname, args, result, - } => { - assert_eq!(opname, "residual_call_r_r"); + } => (opname, args, result), + other => panic!("expected Insn::Op, got {other:?}"), + } + } + + /// Assert the operand tail every single-Ref unary lowering shares, + /// whichever call shape it chose: `ListR([value])` and the Ref result + /// register the fixture asked for. + fn assert_unary_hlop_call_tail(op_name: &str, args: &[Operand], result: Option) { + match &args[1] { + Operand::ListOfKind(list) => { + assert_eq!(list.kind, Kind::Ref); assert!( - matches!(args[0], Operand::ConstInt(idx) if idx == expected_fn_idx), - "{op_name}_fn pool index {expected_fn_idx}, got {:?}", - args[0] - ); - match &args[1] { - Operand::ListOfKind(list) => { - assert_eq!(list.kind, Kind::Ref); - assert!( - matches!(&list.content[..], [Operand::Register(r)] if r.index == 101), - "ListR = [value], got {:?}", - list.content - ); - } - other => panic!("expected ListR, got {other:?}"), - } - assert_eq!( - result, - Some(Register { - kind: Kind::Ref, - index: 102 - }), + matches!(&list.content[..], [Operand::Register(r)] if r.index == 101), + "{op_name} ListR = [value], got {:?}", + list.content ); } - _ => panic!("expected Insn::Op, got {insn:?}"), + other => panic!("expected ListR, got {other:?}"), + } + assert_eq!( + result, + Some(Register { + kind: Kind::Ref, + index: 102 + }), + ); + } + + /// A single-Ref unary HLOp with no inline target emits the residual call + /// on its `_fn` pool index. Pinned outright: no callee path is consulted. + fn assert_unary_lowering_emits_residual( + op_name: &str, + expected_fn_idx: i64, + lower: impl FnOnce( + &super::super::flow::SpaceOperation, + &LoweringContext, + &mut dyn FnMut(Variable) -> Register, + &mut dyn FnMut(&Constant) -> Operand, + ) -> Option, + ) { + let (opname, args, result) = lower_unary_hlop_fixture(op_name, lower); + assert_eq!(opname, "residual_call_r_r"); + assert!( + matches!(args[0], Operand::ConstInt(idx) if idx == expected_fn_idx), + "{op_name}_fn pool index {expected_fn_idx}, got {:?}", + args[0] + ); + assert_unary_hlop_call_tail(op_name, &args, result); + } + + /// A single-Ref unary HLOp whose callee body this build carries fully + /// bound emits the canonical `inline_call_r_r` naming that body; one whose + /// body is absent or still unbound keeps the residual fallback. + /// + /// Which of the two the build offers is a property of the build, so the + /// expectation is read from the callee registry and the body scan rather + /// than from `fully_bound_callee_body`: asking the gate itself would agree + /// with any answer it gave. Read this way, a gate that declines a body + /// that resolves and names no unbound symbolic residual fails here. + fn assert_unary_lowering_inlines_bound_body( + op_name: &str, + canonical_path: &'static str, + expected_fn_idx: i64, + lower: impl FnOnce( + &super::super::flow::SpaceOperation, + &LoweringContext, + &mut dyn FnMut(Variable) -> Register, + &mut dyn FnMut(&Constant) -> Operand, + ) -> Option, + ) { + let (opname, args, result) = lower_unary_hlop_fixture(op_name, lower); + // The scan follows `inline_call_*` through the shared build-time descr + // pool, so this read needs the same installed pool the gate does. + pyre_jit_trace::jitcode_runtime::install_global_build_descr_pool(); + let body_is_bound = + pyre_jit_trace::jitcode_runtime::pathed_runtime_jitcode_cached(canonical_path) + .is_some_and(|jitcode| { + jitcode.startpoints.is_some() + && jitcode.reachable_symbolic_residuals().targets.is_empty() + }); + if body_is_bound { + assert_eq!(opname, "inline_call_r_r"); + assert!( + matches!(args[0], Operand::Descr(ref descr) if matches!(&**descr, DescrOperand::JitCode(_))), + "{op_name} inline target, got {:?}", + args[0] + ); + } else { + assert_eq!(opname, "residual_call_r_r"); + assert!( + matches!(args[0], Operand::ConstInt(idx) if idx == expected_fn_idx), + "{op_name}_fn pool index {expected_fn_idx}, got {:?}", + args[0] + ); } + assert_unary_hlop_call_tail(op_name, &args, result); } #[test] - fn lower_unary_invert_hlop_emits_unary_invert_fn_residual() { - // MayForce — a user `__invert__` may run Python. - assert_unary_lowering_emits_residual("invert", 109, |op, ctx, gr, lc| { - super::lower_unary_invert_hlop_to_insn(op, ctx, &mut |v| gr(v), &mut |c| lc(c)) - }); + fn lower_unary_negative_hlop_emits_inline_call_or_residual_fallback() { + assert_unary_lowering_inlines_bound_body( + "neg", + "pyre_interpreter::objspace::descroperation::neg", + 112, + |op, ctx, gr, lc| { + super::lower_unary_negative_hlop_to_insn(op, ctx, &mut |v| gr(v), &mut |c| lc(c)) + }, + ); } #[test] - fn lower_unary_positive_hlop_emits_unary_positive_fn_residual() { - // MayForce — a user `__pos__` may run Python. - assert_unary_lowering_emits_residual("pos", 118, |op, ctx, gr, lc| { - super::lower_unary_positive_hlop_to_insn(op, ctx, &mut |v| gr(v), &mut |c| lc(c)) - }); + fn lower_unary_invert_hlop_emits_inline_call_or_residual_fallback() { + assert_unary_lowering_inlines_bound_body( + "invert", + "pyre_interpreter::objspace::descroperation::invert", + 109, + |op, ctx, gr, lc| { + super::lower_unary_invert_hlop_to_insn(op, ctx, &mut |v| gr(v), &mut |c| lc(c)) + }, + ); + } + + #[test] + fn lower_unary_positive_hlop_emits_inline_call_or_residual_fallback() { + assert_unary_lowering_inlines_bound_body( + "pos", + "pyre_interpreter::objspace::descroperation::pos", + 118, + |op, ctx, gr, lc| { + super::lower_unary_positive_hlop_to_insn(op, ctx, &mut |v| gr(v), &mut |c| lc(c)) + }, + ); } #[test] diff --git a/pyre/pyre-jit/src/jit/liveness.rs b/pyre/pyre-jit/src/jit/liveness.rs index 054aef52e66..1701b51c431 100644 --- a/pyre/pyre-jit/src/jit/liveness.rs +++ b/pyre/pyre-jit/src/jit/liveness.rs @@ -251,7 +251,8 @@ fn _compute_liveness_must_continue( follow_label(&mut alive, label2alive, label); } } - DescrOperand::Bh(_) + DescrOperand::JitCode(_) + | DescrOperand::Bh(_) | DescrOperand::CallDescrStub(_) | DescrOperand::VableArrayField(_) | DescrOperand::VableArray(_) diff --git a/pyre/pyre-jit/tests/gc_stress.rs b/pyre/pyre-jit/tests/gc_stress.rs index ad130c5bb06..753b40d753c 100644 --- a/pyre/pyre-jit/tests/gc_stress.rs +++ b/pyre/pyre-jit/tests/gc_stress.rs @@ -39,6 +39,59 @@ use pyre_jit::eval::{eval_with_jit, init_jit_hooks, reset_gc_fresh_for_test}; static GC_STRESS_SERIAL: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); +/// `ResumeGuardForcedDescr.handle_async_forcing` runs before the residual +/// returns. Its virtualizable image must not read the pending return value +/// from a register home that compiled code has not written yet. +#[test] +fn pickle_async_forcing_keeps_pending_result_out_of_frame() { + const CHILD: &str = "PYRE_PICKLE_ASYNC_FORCE_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + // Set nursery poisoning at process startup, not by mutating the + // environment underneath the other Rust test workers. The ordinary + // nursery can accidentally give an unwritten result home a zero. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "pickle_async_forcing_keeps_pending_result_out_of_frame", + "--nocapture", + ]) + .env(CHILD, "1") + .env("PYPY_GC_NURSERY_DEBUG", "1") + .output() + .expect("run isolated async-forcing regression"); + assert!( + output.status.success(), + "async forcing read an unavailable call result:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + return; + } + // test_bytes makes PyPicklerTests.loads hot. NEWOBJ_EX subsequently + // enters Python from its residual Unpickler.load and forces that frame. + // Collect after loads has returned; before the fix, + // its old locals array contained nursery poison above valuestackdepth. + run_on_worker( + r#" +import gc +try: + from test.test_pickle import CPicklerTests + + case = CPicklerTests('test_bytes') + case.test_bytes() + case.test_complex_newobj_ex() + gc.collect() +except BaseException: + import traceback + traceback.print_exc() + raise +"#, + "pickle_async_forcing.py", + "async-forcing frame slots", + "pickle async forcing must preserve a valid frame image", + ); +} + /// Shared harness body for every GC-stress program. Compiles and runs `program` /// (using `name` as its `sys.argv[0]` / filename) exactly as the `pyrex` /// launcher would, after installing a fresh per-worker GC heap. An uncaught diff --git a/pyre/pyre-object/src/bytes_array.rs b/pyre/pyre-object/src/bytes_array.rs index d358597eafe..e7175af5b8a 100644 --- a/pyre/pyre-object/src/bytes_array.rs +++ b/pyre/pyre-object/src/bytes_array.rs @@ -1,4 +1,5 @@ use std::ops::{Index, IndexMut}; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::bytesobject::BytesBlock; use crate::object_array::{ @@ -16,13 +17,41 @@ use crate::pyobject::PyObjectRef; #[repr(C)] pub struct BytesArray { pub block: *mut ItemsBlock, - len: usize, + /// Live length (rlist.py `("length", Signed)`), read WITHOUT a lock. + /// + /// `Include/cpython/listobject.h PyList_GET_SIZE` answers a length under + /// `Py_GIL_DISABLED` as `_Py_atomic_load_ssize_relaxed(&ob_size)` — a + /// relaxed atomic load, no critical section — so a reader is entitled to a + /// value from either side of a concurrent mutation but never to a torn one. + /// A compiled trace reads this slot at a raw offset, through the + /// `bytes_items.len` descriptor addressed by [`BYTES_ARRAY_LEN_OFFSET`], which is why it cannot be + /// a plain `usize`: the methods below write it while a compiled loop is + /// reading it, and only an atomic makes that pair defined. Every write + /// here is a relaxed store for the same reason — `&mut self` bounds no + /// raw-pointer reader, so `get_mut()` would put the plain store straight + /// back. + /// + /// Same size and bit validity as `usize`, so `offset_of!` and the JIT's + /// `Type::Int` read are unchanged. + pub(crate) len: AtomicUsize, } pub const BYTES_ARRAY_BLOCK_OFFSET: usize = std::mem::offset_of!(BytesArray, block); pub const BYTES_ARRAY_LEN_OFFSET: usize = std::mem::offset_of!(BytesArray, len); impl BytesArray { + /// `_Py_atomic_load_ssize_relaxed(&ob_size)`. + #[inline] + fn len_relaxed(&self) -> usize { + self.len.load(Ordering::Relaxed) + } + + /// `_Py_atomic_store_ssize_relaxed(&ob_size, n)`. + #[inline] + fn set_len_relaxed(&self, n: usize) { + self.len.store(n, Ordering::Relaxed); + } + #[inline] fn base(&self) -> *mut PyObjectRef { unsafe { items_block_items_base(self.block) } @@ -31,7 +60,7 @@ impl BytesArray { pub fn empty() -> Self { Self { block: std::ptr::null_mut(), - len: 0, + len: AtomicUsize::new(0), } } @@ -43,7 +72,7 @@ impl BytesArray { let len = refs.len(); Self { block: unsafe { alloc_list_items_block_gc(&refs) }, - len, + len: AtomicUsize::new(len), } } @@ -56,7 +85,7 @@ impl BytesArray { block: unsafe { crate::object_array::grow_list_items_block_gc(std::ptr::null_mut(), capacity, 0) }, - len: 0, + len: AtomicUsize::new(0), } } @@ -85,7 +114,7 @@ impl BytesArray { #[inline] pub fn spare_capacity(&self) -> usize { - self.capacity().saturating_sub(self.len) + self.capacity().saturating_sub(self.len_relaxed()) } #[inline] @@ -96,7 +125,7 @@ impl BytesArray { #[inline] pub fn set_len(&mut self, new_len: usize) { assert!(new_len <= self.capacity()); - self.len = new_len; + self.set_len_relaxed(new_len); } #[inline] @@ -118,7 +147,7 @@ impl BytesArray { #[inline] fn assert_room(&self, additional: usize) { assert!( - self.len + additional <= self.capacity(), + self.len_relaxed() + additional <= self.capacity(), "BytesArray needs {additional} more slot(s) than its capacity {}; \ reserve through W_ListObject::bytes_grow first", self.capacity(), @@ -152,26 +181,35 @@ impl BytesArray { let _ = crate::gc_roots::pin_root(value as PyObjectRef); self.assert_room(1); self.barrier(); - unsafe { *self.base().add(self.len) = crate::gc_roots::shadow_stack_get(value_slot) }; - self.len += 1; + unsafe { + *self.base().add(self.len_relaxed()) = crate::gc_roots::shadow_stack_get(value_slot) + }; + self.set_len_relaxed(self.len_relaxed() + 1); } #[inline] pub fn len(&self) -> usize { - self.len + self.len_relaxed() } #[inline] pub fn is_empty(&self) -> bool { - self.len == 0 + self.len_relaxed() == 0 } pub fn as_slice(&self) -> &[*const BytesBlock] { - unsafe { std::slice::from_raw_parts(self.base() as *const *const BytesBlock, self.len) } + unsafe { + std::slice::from_raw_parts(self.base() as *const *const BytesBlock, self.len_relaxed()) + } } pub fn as_mut_slice(&mut self) -> &mut [*const BytesBlock] { - unsafe { std::slice::from_raw_parts_mut(self.base() as *mut *const BytesBlock, self.len) } + unsafe { + std::slice::from_raw_parts_mut( + self.base() as *mut *const BytesBlock, + self.len_relaxed(), + ) + } } pub fn to_vec(&self) -> Vec<*const BytesBlock> { @@ -179,7 +217,7 @@ impl BytesArray { } pub fn insert(&mut self, index: usize, value: *const BytesBlock) { - assert!(index <= self.len); + assert!(index <= self.len_relaxed()); let _roots = crate::gc_roots::push_roots(); let value_slot = crate::gc_roots::shadow_stack_len(); let _ = crate::gc_roots::pin_root(value as PyObjectRef); @@ -188,14 +226,14 @@ impl BytesArray { self.before_move_barrier(); unsafe { let p = self.base().add(index); - std::ptr::copy(p, p.add(1), self.len - index); + std::ptr::copy(p, p.add(1), self.len_relaxed() - index); *p = crate::gc_roots::shadow_stack_get(value_slot); } - self.len += 1; + self.set_len_relaxed(self.len_relaxed() + 1); } pub fn set(&mut self, index: usize, value: *const BytesBlock) { - assert!(index < self.len); + assert!(index < self.len_relaxed()); let _roots = crate::gc_roots::push_roots(); let slot = crate::gc_roots::shadow_stack_len(); let _ = crate::gc_roots::pin_root(value as PyObjectRef); @@ -204,23 +242,23 @@ impl BytesArray { } pub fn remove(&mut self, index: usize) -> *const BytesBlock { - assert!(index < self.len); + assert!(index < self.len_relaxed()); let value = self.as_slice()[index]; self.before_move_barrier(); unsafe { let p = self.base().add(index); - std::ptr::copy(p.add(1), p, self.len - index - 1); - *p.add(self.len - index - 1) = std::ptr::null_mut(); + std::ptr::copy(p.add(1), p, self.len_relaxed() - index - 1); + *p.add(self.len_relaxed() - index - 1) = std::ptr::null_mut(); } - self.len -= 1; + self.set_len_relaxed(self.len_relaxed() - 1); value } pub fn pop(&mut self) -> *const BytesBlock { - assert!(self.len > 0); - let value = self.as_slice()[self.len - 1]; - self.len -= 1; - unsafe { *self.base().add(self.len) = std::ptr::null_mut() }; + assert!(self.len_relaxed() > 0); + let value = self.as_slice()[self.len_relaxed() - 1]; + self.set_len_relaxed(self.len_relaxed() - 1); + unsafe { *self.base().add(self.len_relaxed()) = std::ptr::null_mut() }; value } @@ -229,7 +267,7 @@ impl BytesArray { } pub fn splice(&mut self, start: usize, remove_count: usize, values: &[*const BytesBlock]) { - let old_len = self.len; + let old_len = self.len_relaxed(); let start = start.min(old_len); let removed = remove_count.min(old_len - start); let new_len = old_len - removed + values.len(); @@ -253,7 +291,7 @@ impl BytesArray { base.add(start + values.len()), old_len - start - removed, ); - self.len = new_len; + self.set_len_relaxed(new_len); for i in 0..values.len() { *base.add(start + i) = crate::gc_roots::shadow_stack_get(root_base + i); } @@ -264,7 +302,7 @@ impl BytesArray { } pub fn drain(&mut self, range: std::ops::Range) { - assert!(range.start <= range.end && range.end <= self.len); + assert!(range.start <= range.end && range.end <= self.len_relaxed()); let count = range.end - range.start; if count == 0 { return; @@ -275,22 +313,22 @@ impl BytesArray { std::ptr::copy( base.add(range.end), base.add(range.start), - self.len - range.end, + self.len_relaxed() - range.end, ); - for i in self.len - count..self.len { + for i in self.len_relaxed() - count..self.len_relaxed() { *base.add(i) = std::ptr::null_mut(); } } - self.len -= count; + self.set_len_relaxed(self.len_relaxed() - count); } pub fn clear(&mut self) { unsafe { - for i in 0..self.len { + for i in 0..self.len_relaxed() { *self.base().add(i) = std::ptr::null_mut(); } } - self.len = 0; + self.set_len_relaxed(0); } } diff --git a/pyre/pyre-object/src/float_array.rs b/pyre/pyre-object/src/float_array.rs index b514d5fe4b1..b3f19384e04 100644 --- a/pyre/pyre-object/src/float_array.rs +++ b/pyre/pyre-object/src/float_array.rs @@ -39,7 +39,7 @@ pub struct FloatArray { /// /// Same size and bit validity as `usize`, so `offset_of!` and the JIT's /// `Type::Int` read are unchanged. - len: AtomicUsize, + pub(crate) len: AtomicUsize, } pub const FLOAT_ARRAY_BLOCK_OFFSET: usize = std::mem::offset_of!(FloatArray, block); diff --git a/pyre/pyre-object/src/int_array.rs b/pyre/pyre-object/src/int_array.rs index 8f2d9390a87..e0e7e7418e3 100644 --- a/pyre/pyre-object/src/int_array.rs +++ b/pyre/pyre-object/src/int_array.rs @@ -43,7 +43,7 @@ pub struct IntArray { /// /// Same size and bit validity as `usize`, so `offset_of!` and the JIT's /// `Type::Int` read are unchanged. - len: AtomicUsize, + pub(crate) len: AtomicUsize, } pub const INT_ARRAY_BLOCK_OFFSET: usize = std::mem::offset_of!(IntArray, block); diff --git a/pyre/pyre-object/src/interp_exceptions.rs b/pyre/pyre-object/src/interp_exceptions.rs index 9eb29dfe39a..565538deada 100644 --- a/pyre/pyre-object/src/interp_exceptions.rs +++ b/pyre/pyre-object/src/interp_exceptions.rs @@ -832,11 +832,14 @@ pub fn register_exc_class_for_kind(kind: ExcKind, cls: PyObjectRef) -> PyObjectR } } -/// Reads the process-global `EXC_CLASS_BY_KIND`, a runtime-mutable root the -/// tracer cannot type; the JIT residualises the read instead of tracing into -/// it (`@dont_look_inside`, `rlib/jit.py`). The residual call resolves its -/// address by qualified path in `jit_trace_fnaddrs`. -#[majit_macros::dont_look_inside] +/// Reads the process-global `EXC_CLASS_BY_KIND`, a root the tracer cannot +/// type, so the JIT residualises the read. It is elidable: a slot is written +/// once, by `register_exc_class_for_kind`'s first writer, and an instance of +/// a kind exists only after its class was registered (the internal raise paths +/// build instances of the init-time kinds only), so a read made for a live +/// instance never observes the slot's empty state. The residual call resolves +/// its address by qualified path in `jit_trace_fnaddrs`. +#[majit_macros::elidable] pub fn lookup_exc_class_for_kind(kind: ExcKind) -> PyObjectRef { EXC_CLASS_BY_KIND[kind as u8 as usize].load(std::sync::atomic::Ordering::Acquire) as PyObjectRef } diff --git a/pyre/pyre-object/src/listobject.rs b/pyre/pyre-object/src/listobject.rs index 00cf2f60507..7de3afe36d8 100644 --- a/pyre/pyre-object/src/listobject.rs +++ b/pyre/pyre-object/src/listobject.rs @@ -310,12 +310,15 @@ impl W_ListObject { ListStrategy::Empty | ListStrategy::Size => 0, ListStrategy::SimpleRange | ListStrategy::Range => unsafe { range_list_length(self) }, ListStrategy::Object => self.length_relaxed(), - // Direct rlist `length` field reads keep this helper in the - // annotator's structural subset; the public `.len()` wrappers are - // host collection conveniences that translate as `__len__`. - ListStrategy::Integer => self.int_items.len(), - ListStrategy::IntOrFloat => self.int_items.len(), - ListStrategy::Float => self.float_items.len(), + // rlist.py `ll_length` is the `list.int_len` / `list.float_len` + // oopspec leaf. Keep that call boundary: jtransform lowers the + // nested `int_items.len` / `float_items.len` path to one field read + // off the list owner. Inlining the Rust field path instead first + // materialises the by-value storage struct as a GC Ref and then + // reads through it, which is neither an RPython getsubstruct nor a + // valid pointer. + ListStrategy::Integer | ListStrategy::IntOrFloat => ll_list_int_length(self), + ListStrategy::Float => ll_list_float_length(self), ListStrategy::Bytes => self.bytes_items.len(), ListStrategy::Ascii => self.ascii_items.len(), } @@ -2905,28 +2908,26 @@ pub unsafe fn w_list_int_or_float_setitem( true } -/// Get the length of a list. +/// Get the length of a list: `W_ListObject.length()`, a strategy-dispatched +/// field read (`listobject.py EmptyListStrategy.length` returns 0). +/// +/// Read without the list's lock, the way `PyList_GET_SIZE` reads `ob_size` +/// and the way the walked `w_list_*_inner` bodies already read it. What +/// makes that sound is the GIL (`majit-gc` `rgil`), which serialises every +/// mutator against this read: the lock is a narrower scope inside it, not +/// the thing keeping a strategy switch from being observed half-applied. +/// The distinction matters because a half-applied switch is not merely a +/// stale length — the two range strategies answer through `list.items`, and +/// `switch_range_to_integer_strategy` nulls that pointer before it stores +/// the new strategy. With no lock there is no collection point, so `obj` +/// needs no root here. Taking the lock instead made the acquire an opaque +/// residual (`w_list_lock` is `dont_look_inside`) on every traced +/// `len(list)`, which is what kept the generic builtin descent out of `len`. /// /// # Safety /// `obj` must point to a valid `W_ListObject`. pub unsafe fn w_list_len(obj: PyObjectRef) -> usize { - let _roots = crate::gc_roots::push_roots(); - let root_base = crate::gc_roots::shadow_stack_len(); - let obj = crate::gc_roots::pin_root(obj); - let _list_guard = w_list_lock(obj); - let obj = crate::gc_roots::shadow_stack_get(root_base); - let list = &*(obj as *const W_ListObject); - match list.strategy { - // listobject.py EmptyListStrategy.length returns 0. - ListStrategy::Empty | ListStrategy::Size => 0, - ListStrategy::SimpleRange | ListStrategy::Range => range_list_length(list), - ListStrategy::Object => list.length_relaxed(), - ListStrategy::Integer => ll_list_int_length(list), - ListStrategy::IntOrFloat => list.int_items.len(), - ListStrategy::Float => list.float_items.len(), - ListStrategy::Bytes => list.bytes_items.len(), - ListStrategy::Ascii => list.ascii_items.len(), - } + (*(obj as *const W_ListObject)).live_len() } /// CPython-visible `PyListObject.allocated` under the list's mutation lock. diff --git a/pyre/pyre-object/src/longobject.rs b/pyre/pyre-object/src/longobject.rs index 384a84ba5ae..4df8067d8aa 100644 --- a/pyre/pyre-object/src/longobject.rs +++ b/pyre/pyre-object/src/longobject.rs @@ -577,15 +577,6 @@ pub extern "C" fn jit_bigint_add(a: i64, b: i64) -> i64 { } } -/// `rbigint.add_int_int_bigint_result` (`rpython/rlib/rbigint.py`, -/// `@jit.elidable`) — exact bigint sum of two machine ints. Allocates the -/// result via the COLLECTING nursery, matching [`jit_bigint_add`], and returns -/// a freshly heap-allocated `*mut BigInt` payload (as i64). -#[majit_macros::elidable_or_memerror] -pub extern "C" fn jit_bigint_add_int_int(a: i64, b: i64) -> i64 { - alloc_bigint_nursery_collecting(BigInt::add_int_int_bigint_result(a, b)) as i64 -} - /// `rbigint.sub` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_sub(a: i64, b: i64) -> i64 { @@ -600,14 +591,6 @@ pub extern "C" fn jit_bigint_sub(a: i64, b: i64) -> i64 { } } -/// `rbigint.sub_int_int_bigint_result` (`rpython/rlib/rbigint.py`, -/// `@jit.elidable`) — exact bigint difference of two machine ints. See -/// [`jit_bigint_add_int_int`]. -#[majit_macros::elidable_or_memerror] -pub extern "C" fn jit_bigint_sub_int_int(a: i64, b: i64) -> i64 { - alloc_bigint_nursery_collecting(BigInt::sub_int_int_bigint_result(a, b)) as i64 -} - /// `rbigint.mul` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_mul(a: i64, b: i64) -> i64 { @@ -615,14 +598,6 @@ pub extern "C" fn jit_bigint_mul(a: i64, b: i64) -> i64 { unsafe { alloc_bigint_nursery_collecting(&*a * &*b) as i64 } } -/// `rbigint.mul_int_int_bigint_result` (`rpython/rlib/rbigint.py`, -/// `@jit.elidable`) — exact bigint product of two machine ints. See -/// [`jit_bigint_add_int_int`]. -#[majit_macros::elidable_or_memerror] -pub extern "C" fn jit_bigint_mul_int_int(a: i64, b: i64) -> i64 { - alloc_bigint_nursery_collecting(BigInt::mul_int_int_bigint_result(a, b)) as i64 -} - /// `rbigint.and_` on bare payloads (collecting). See [`jit_bigint_add`]. #[majit_macros::elidable_or_memerror] pub extern "C" fn jit_bigint_and(a: i64, b: i64) -> i64 { diff --git a/pyre/pyre-object/src/pyobject.rs b/pyre/pyre-object/src/pyobject.rs index eb27a051ac0..00a02296cd6 100644 --- a/pyre/pyre-object/src/pyobject.rs +++ b/pyre/pyre-object/src/pyobject.rs @@ -140,8 +140,10 @@ pub fn get_instantiate(tp: &PyType) -> PyObjectRef { /// and let a subclass fall through to the MRO `lookup` path. /// /// A fresh builtin carries `w_class == get_instantiate(ob_type)` (see -/// `w_int_new` etc.); the read-only singletons (`True` / `False` / `None` / -/// `Ellipsis` / `NotImplemented`) leave `w_class` null and are always exact. +/// `w_int_new` etc.); the specialised arity-2 tuple layouts instead carry the +/// canonical `tuple` class, exactly as [`is_exact_tuple`] requires. The +/// read-only singletons (`True` / `False` / `None` / `Ellipsis` / +/// `NotImplemented`) leave `w_class` null and are always exact. /// /// # Safety /// `obj` must be null or a valid `PyObjectRef`. @@ -158,7 +160,22 @@ pub unsafe fn is_exact_builtin_instance(obj: PyObjectRef) -> bool { } unsafe { let w_class = (*obj).w_class; - w_class.is_null() || std::ptr::eq(w_class, get_instantiate(&*(*obj).ob_type)) + if w_class.is_null() { + return true; + } + let ob_type = (*obj).ob_type; + use crate::specialisedtupleobject::{ + SPECIALISED_TUPLE_FF_TYPE, SPECIALISED_TUPLE_II_TYPE, SPECIALISED_TUPLE_OO_TYPE, + }; + let builtin_class = if std::ptr::eq(ob_type, &SPECIALISED_TUPLE_II_TYPE) + || std::ptr::eq(ob_type, &SPECIALISED_TUPLE_FF_TYPE) + || std::ptr::eq(ob_type, &SPECIALISED_TUPLE_OO_TYPE) + { + get_instantiate(&TUPLE_TYPE) + } else { + get_instantiate(&*ob_type) + }; + std::ptr::eq(w_class, builtin_class) } } @@ -168,8 +185,7 @@ pub unsafe fn is_exact_builtin_instance(obj: PyObjectRef) -> bool { /// Unlike [`is_exact_builtin_instance`] this is correct for the specialised /// arity-2 tuples: they carry a distinct `ob_type` /// (`SPECIALISED_TUPLE_*_TYPE`) but a `w_class` of the canonical `tuple` type -/// object, so `is_exact_type(t, &TUPLE_TYPE)` is `true` for them while -/// `is_exact_builtin_instance` (which keys off `ob_type`) is not. A user +/// object, so `is_exact_type(t, &TUPLE_TYPE)` is `true` for them. A user /// subclass retags `w_class` to its own type object and so is rejected. /// /// # Safety @@ -892,11 +908,13 @@ pub fn compute_subclass_ranges_from(alias_chains: &[&[SubclassRangeAlias]]) { /// differs. A later GC writeback is byte-identical. static SUBCLASS_RANGES_INIT: OnceLock<()> = OnceLock::new(); -// `dont_look_inside`: one-time host initialization (`OnceLock` + -// global type-table walk) stays opaque to the JIT — production -// entry points have run the full init before any trace executes, -// so the residual call is a no-op there. -#[majit_macros::dont_look_inside] +// `not_in_trace`: one-time host initialization (`OnceLock` + global +// type-table walk). It still runs while tracing and blackholing, and +// production entry points have run the full init before any trace +// executes, so compiled code omits the call: as a residual it was one +// effectful call — and a heap-cache flush — on every traced +// `is_exception` probe. +#[majit_macros::not_in_trace] pub extern "C" fn ensure_object_subclass_ranges_initialized() { SUBCLASS_RANGES_INIT.get_or_init(|| { let aliases = all_subclass_range_aliases(); diff --git a/pyre/pyre-object/src/tupleobject.rs b/pyre/pyre-object/src/tupleobject.rs index eda0a5ca152..0a8fb926e7e 100644 --- a/pyre/pyre-object/src/tupleobject.rs +++ b/pyre/pyre-object/src/tupleobject.rs @@ -675,10 +675,20 @@ unsafe fn w_tuple_getitem_known(obj: PyObjectRef, idx: usize) -> PyObjectRef { /// # Safety /// `obj` must point to a valid tuple of any of the four variants. pub unsafe fn w_tuple_len(obj: PyObjectRef) -> usize { - if is_specialised_tuple_ii(obj) || is_specialised_tuple_ff(obj) || is_specialised_tuple_oo(obj) + // specialisedtupleobject.py's three generated classes each implement + // `length()` as the constant `typelen`. Dispatch on the payload vtable + // directly, as `w_tuple_getitem_known` does below: routing this through + // `py_type_check` makes the generated walk mix the user-visible canonical + // tuple class with the three payload layouts and can fall through to a + // `W_TupleObject.wrappeditems` read on an inline specialised payload. + let ob_type = unsafe { (*obj).ob_type }; + if std::ptr::eq(ob_type, &SPECIALISED_TUPLE_II_TYPE) + || std::ptr::eq(ob_type, &SPECIALISED_TUPLE_FF_TYPE) + || std::ptr::eq(ob_type, &SPECIALISED_TUPLE_OO_TYPE) { return 2; } + debug_assert!(std::ptr::eq(ob_type, &TUPLE_TYPE)); let tuple = &*(obj as *const W_TupleObject); items_block_capacity(tuple.wrappeditems) } diff --git a/pyre/pyre-object/src/unicode_array.rs b/pyre/pyre-object/src/unicode_array.rs index e82a8d16078..57d93e8a5af 100644 --- a/pyre/pyre-object/src/unicode_array.rs +++ b/pyre/pyre-object/src/unicode_array.rs @@ -1,4 +1,5 @@ use std::ops::{Index, IndexMut}; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::object_array::{ ItemsBlock, alloc_list_items_block_gc, dealloc_list_items_block, items_block_capacity, @@ -16,13 +17,41 @@ use rustpython_wtf8::Wtf8Buf; #[repr(C)] pub struct UnicodeArray { pub block: *mut ItemsBlock, - len: usize, + /// Live length (rlist.py `("length", Signed)`), read WITHOUT a lock. + /// + /// `Include/cpython/listobject.h PyList_GET_SIZE` answers a length under + /// `Py_GIL_DISABLED` as `_Py_atomic_load_ssize_relaxed(&ob_size)` — a + /// relaxed atomic load, no critical section — so a reader is entitled to a + /// value from either side of a concurrent mutation but never to a torn one. + /// A compiled trace reads this slot at a raw offset, through the + /// `ascii_items.len` descriptor addressed by [`UNICODE_ARRAY_LEN_OFFSET`], which is why it cannot be + /// a plain `usize`: the methods below write it while a compiled loop is + /// reading it, and only an atomic makes that pair defined. Every write + /// here is a relaxed store for the same reason — `&mut self` bounds no + /// raw-pointer reader, so `get_mut()` would put the plain store straight + /// back. + /// + /// Same size and bit validity as `usize`, so `offset_of!` and the JIT's + /// `Type::Int` read are unchanged. + pub(crate) len: AtomicUsize, } pub const UNICODE_ARRAY_BLOCK_OFFSET: usize = std::mem::offset_of!(UnicodeArray, block); pub const UNICODE_ARRAY_LEN_OFFSET: usize = std::mem::offset_of!(UnicodeArray, len); impl UnicodeArray { + /// `_Py_atomic_load_ssize_relaxed(&ob_size)`. + #[inline] + fn len_relaxed(&self) -> usize { + self.len.load(Ordering::Relaxed) + } + + /// `_Py_atomic_store_ssize_relaxed(&ob_size, n)`. + #[inline] + fn set_len_relaxed(&self, n: usize) { + self.len.store(n, Ordering::Relaxed); + } + #[inline] fn base(&self) -> *mut PyObjectRef { unsafe { items_block_items_base(self.block) } @@ -31,7 +60,7 @@ impl UnicodeArray { pub fn empty() -> Self { Self { block: std::ptr::null_mut(), - len: 0, + len: AtomicUsize::new(0), } } @@ -43,7 +72,7 @@ impl UnicodeArray { let len = refs.len(); Self { block: unsafe { alloc_list_items_block_gc(&refs) }, - len, + len: AtomicUsize::new(len), } } @@ -56,7 +85,7 @@ impl UnicodeArray { block: unsafe { crate::object_array::grow_list_items_block_gc(std::ptr::null_mut(), capacity, 0) }, - len: 0, + len: AtomicUsize::new(0), } } @@ -85,7 +114,7 @@ impl UnicodeArray { #[inline] pub fn spare_capacity(&self) -> usize { - self.capacity().saturating_sub(self.len) + self.capacity().saturating_sub(self.len_relaxed()) } #[inline] @@ -96,7 +125,7 @@ impl UnicodeArray { #[inline] pub fn set_len(&mut self, new_len: usize) { assert!(new_len <= self.capacity()); - self.len = new_len; + self.set_len_relaxed(new_len); } #[inline] @@ -118,7 +147,7 @@ impl UnicodeArray { #[inline] fn assert_room(&self, additional: usize) { assert!( - self.len + additional <= self.capacity(), + self.len_relaxed() + additional <= self.capacity(), "UnicodeArray needs {additional} more slot(s) than its capacity {}; \ reserve through W_ListObject::ascii_grow first", self.capacity(), @@ -152,26 +181,32 @@ impl UnicodeArray { let _ = crate::gc_roots::pin_root(value as PyObjectRef); self.assert_room(1); self.barrier(); - unsafe { *self.base().add(self.len) = crate::gc_roots::shadow_stack_get(value_slot) }; - self.len += 1; + unsafe { + *self.base().add(self.len_relaxed()) = crate::gc_roots::shadow_stack_get(value_slot) + }; + self.set_len_relaxed(self.len_relaxed() + 1); } #[inline] pub fn len(&self) -> usize { - self.len + self.len_relaxed() } #[inline] pub fn is_empty(&self) -> bool { - self.len == 0 + self.len_relaxed() == 0 } pub fn as_slice(&self) -> &[*const Wtf8Buf] { - unsafe { std::slice::from_raw_parts(self.base() as *const *const Wtf8Buf, self.len) } + unsafe { + std::slice::from_raw_parts(self.base() as *const *const Wtf8Buf, self.len_relaxed()) + } } pub fn as_mut_slice(&mut self) -> &mut [*const Wtf8Buf] { - unsafe { std::slice::from_raw_parts_mut(self.base() as *mut *const Wtf8Buf, self.len) } + unsafe { + std::slice::from_raw_parts_mut(self.base() as *mut *const Wtf8Buf, self.len_relaxed()) + } } pub fn to_vec(&self) -> Vec<*const Wtf8Buf> { @@ -179,7 +214,7 @@ impl UnicodeArray { } pub fn insert(&mut self, index: usize, value: *const Wtf8Buf) { - assert!(index <= self.len); + assert!(index <= self.len_relaxed()); let _roots = crate::gc_roots::push_roots(); let value_slot = crate::gc_roots::shadow_stack_len(); let _ = crate::gc_roots::pin_root(value as PyObjectRef); @@ -188,14 +223,14 @@ impl UnicodeArray { self.before_move_barrier(); unsafe { let p = self.base().add(index); - std::ptr::copy(p, p.add(1), self.len - index); + std::ptr::copy(p, p.add(1), self.len_relaxed() - index); *p = crate::gc_roots::shadow_stack_get(value_slot); } - self.len += 1; + self.set_len_relaxed(self.len_relaxed() + 1); } pub fn set(&mut self, index: usize, value: *const Wtf8Buf) { - assert!(index < self.len); + assert!(index < self.len_relaxed()); let _roots = crate::gc_roots::push_roots(); let slot = crate::gc_roots::shadow_stack_len(); let _ = crate::gc_roots::pin_root(value as PyObjectRef); @@ -204,23 +239,23 @@ impl UnicodeArray { } pub fn remove(&mut self, index: usize) -> *const Wtf8Buf { - assert!(index < self.len); + assert!(index < self.len_relaxed()); let value = self.as_slice()[index]; self.before_move_barrier(); unsafe { let p = self.base().add(index); - std::ptr::copy(p.add(1), p, self.len - index - 1); - *p.add(self.len - index - 1) = std::ptr::null_mut(); + std::ptr::copy(p.add(1), p, self.len_relaxed() - index - 1); + *p.add(self.len_relaxed() - index - 1) = std::ptr::null_mut(); } - self.len -= 1; + self.set_len_relaxed(self.len_relaxed() - 1); value } pub fn pop(&mut self) -> *const Wtf8Buf { - assert!(self.len > 0); - let value = self.as_slice()[self.len - 1]; - self.len -= 1; - unsafe { *self.base().add(self.len) = std::ptr::null_mut() }; + assert!(self.len_relaxed() > 0); + let value = self.as_slice()[self.len_relaxed() - 1]; + self.set_len_relaxed(self.len_relaxed() - 1); + unsafe { *self.base().add(self.len_relaxed()) = std::ptr::null_mut() }; value } @@ -229,7 +264,7 @@ impl UnicodeArray { } pub fn splice(&mut self, start: usize, remove_count: usize, values: &[*const Wtf8Buf]) { - let old_len = self.len; + let old_len = self.len_relaxed(); let start = start.min(old_len); let removed = remove_count.min(old_len - start); let new_len = old_len - removed + values.len(); @@ -253,7 +288,7 @@ impl UnicodeArray { base.add(start + values.len()), old_len - start - removed, ); - self.len = new_len; + self.set_len_relaxed(new_len); for i in 0..values.len() { *base.add(start + i) = crate::gc_roots::shadow_stack_get(root_base + i); } @@ -264,7 +299,7 @@ impl UnicodeArray { } pub fn drain(&mut self, range: std::ops::Range) { - assert!(range.start <= range.end && range.end <= self.len); + assert!(range.start <= range.end && range.end <= self.len_relaxed()); let count = range.end - range.start; if count == 0 { return; @@ -275,22 +310,22 @@ impl UnicodeArray { std::ptr::copy( base.add(range.end), base.add(range.start), - self.len - range.end, + self.len_relaxed() - range.end, ); - for i in self.len - count..self.len { + for i in self.len_relaxed() - count..self.len_relaxed() { *base.add(i) = std::ptr::null_mut(); } } - self.len -= count; + self.set_len_relaxed(self.len_relaxed() - count); } pub fn clear(&mut self) { unsafe { - for i in 0..self.len { + for i in 0..self.len_relaxed() { *self.base().add(i) = std::ptr::null_mut(); } } - self.len = 0; + self.set_len_relaxed(0); } } diff --git a/pyre/pyre-wasm/src/lib.rs b/pyre/pyre-wasm/src/lib.rs index ef11a4c5c32..b894de88ee5 100644 --- a/pyre/pyre-wasm/src/lib.rs +++ b/pyre/pyre-wasm/src/lib.rs @@ -182,18 +182,18 @@ mod residual_host { unsafe impl Sync for Scratch {} static SCRATCH: Scratch = Scratch(UnsafeCell::new([0u8; SCRATCH_LEN])); - /// Direct-call the blackhole helpers whose real wasm ABI is exactly the - /// uniform `i64` signature carried by the residual call. + /// Direct-call the blackhole helpers whose real wasm ABI is known exactly. /// /// The generic path below must reflect the callee's wasm type in the host: /// an `r` argument may be a real `i32` pointer, a void descriptor may name /// a word-returning target, and guessing either signature traps at a wasm - /// `call_indirect`. These targets are different: every one is declared as - /// an explicit `pub extern "C" fn(i64, ...) -> i64` wrapper, and the CPU - /// function table stores those exact function addresses. Comparing the - /// table index (`fn as usize` on wasm32) therefore proves both the callee - /// identity and its ABI. Calling the named wrapper directly matches the - /// native blackhole dispatch while avoiding a guest -> host -> guest + /// `call_indirect`. These targets are different: each word-returning entry + /// is declared as an explicit `pub extern "C" fn(i64, ...) -> i64` wrapper, + /// while the one true-void initializer is matched separately at arity zero. + /// The CPU function table stores those exact function addresses. Comparing + /// the table index (`fn as usize` on wasm32) therefore proves both the + /// callee identity and its ABI. Calling the named function directly matches + /// the native blackhole dispatch while avoiding a guest -> host -> guest /// reflection round-trip. /// /// Keep this an exact-function allow-list, not a signature inference: the @@ -206,6 +206,13 @@ mod residual_host { /// remaining crossing arrives on this path (`src=host`) rather than from a /// compiled trace; these are its heaviest callees. fn direct_uniform_i64_call(func_ptr: usize, args: &[i64]) -> Option { + if args.is_empty() + && func_ptr == pyre_object::pyobject::ensure_object_subclass_ranges_initialized as usize + { + pyre_object::pyobject::ensure_object_subclass_ranges_initialized(); + return Some(0); + } + macro_rules! uniform_i64_allow_list { ($( [$($arg:ident),*] => $callee:path ),* $(,)?) => { match args { @@ -224,6 +231,7 @@ mod residual_host { [value] => pyre_jit::call_jit::bh_truth_fn, [array] => pyre_jit::call_jit::bh_newtuple_from_array, [array] => pyre_jit::call_jit::bh_newlist_from_array, + [subcls, cls] => pyre_object::pyobject::__majit_call_target_ll_issubclass, [index, seq] => pyre_jit::call_jit::bh_unpack_item_fn, [callable, null_or_self] => pyre_jit::call_jit::bh_call_fn_0, [lhs, rhs, op_code] => pyre_jit::call_jit::bh_binary_op_fn,