diff --git a/Cargo.toml b/Cargo.toml index 374c005ec03..a1b5ba3b9a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -180,6 +180,14 @@ tempfile = "3" walkdir = "2" insta = "1" +# `cargo run --release` is the benchmark/developer performance surface too. +# Give it the same cross-crate optimization used by distributable binaries; +# codegen-units=1 additionally lets LLVM optimize the interpreter's large +# dispatch/call graph as one unit instead of preserving parallel-CGU barriers. +[profile.release] +lto = "thin" +codegen-units = 1 + # The profile that 'dist' will build with [profile.dist] inherits = "release" diff --git a/majit/majit-gc/src/collector.rs b/majit/majit-gc/src/collector.rs index 2b8f8c78e0b..0e80a15b9f6 100644 --- a/majit/majit-gc/src/collector.rs +++ b/majit/majit-gc/src/collector.rs @@ -113,12 +113,19 @@ fn get_darwin_sysctl_signed(name: &[u8]) -> i64 { } } -/// env.py:413-433 `get_L2cache_darwin`. Returns the L2+L3 cache size in -/// bytes via `sysctl`, or -1 when it cannot be determined. +/// env.py:413-455 `get_L2cache_darwin`. Returns the performance-cluster +/// L2 plus the legacy L3 cache size via `sysctl`, or -1 when it cannot be +/// determined. Apple documents lower performance-level indices as faster +/// cores, so `hw.perflevel0.l2cachesize` is the cache relevant to the cores +/// running the mutator. Intel Macs do not expose that key and retain the +/// legacy `hw.l2cachesize` fallback. #[cfg(target_os = "macos")] fn get_l2cache() -> i64 { - let mangled = get_darwin_sysctl_signed(b"hw.l2cachesize\0") - + get_darwin_sysctl_signed(b"hw.l3cachesize\0"); + let mut l2cache = get_darwin_sysctl_signed(b"hw.perflevel0.l2cachesize\0"); + if l2cache <= 0 { + l2cache = get_darwin_sysctl_signed(b"hw.l2cachesize\0"); + } + let mangled = l2cache + get_darwin_sysctl_signed(b"hw.l3cachesize\0"); if mangled > 0 { mangled } else { -1 } } diff --git a/majit/majit-ir/src/eval_breaker_word.rs b/majit/majit-ir/src/eval_breaker_word.rs index 70e8565d812..7b39081b04a 100644 --- a/majit/majit-ir/src/eval_breaker_word.rs +++ b/majit/majit-ir/src/eval_breaker_word.rs @@ -5,6 +5,12 @@ //! OR'd in by the OS signal handler and the action dispatcher. //! bit1 EB_STW — mirrors `GC_SYNC.stw_requested`; OR'd in by the collector //! while it drains mutators to safepoints. +//! bit2 EB_FINALIZING — mirrors interpreter finalization; once armed, +//! non-owner mutators park before their next opcode. +//! bit3 EB_GC_INTERP — process-stable `PYRE_GC_INTERP` dispatch gate. This +//! is masked out of compiled back-edge polls: it avoids +//! a second per-opcode atomic load in the interpreter, +//! but is not itself a reason to leave machine code. //! A compiled loop loads the whole word at the back-edge and deopts to the //! interpreter when it is non-zero. The interpreter/warm-up loop and the STW //! park gate remain authoritative; this word is only the JIT's deopt trigger. @@ -24,6 +30,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub const EB_ASYNC: usize = 1; /// bit1 — GC stop-the-world requested (mirrors `GC_SYNC.stw_requested`). pub const EB_STW: usize = 2; +/// bit2 — interpreter finalization has begun (terminal, never cleared). +pub const EB_FINALIZING: usize = 4; +/// bit3 — interpreter-path allocation/collection integration is enabled. +pub const EB_GC_INTERP: usize = 8; +/// Bits that require a compiled loop to deopt to the interpreter. +pub const JIT_BREAKER_MASK: usize = EB_ASYNC | EB_STW | EB_FINALIZING; /// The shared eval-breaker word (see module docs). static EVAL_BREAKER_WORD: AtomicUsize = AtomicUsize::new(0); @@ -75,7 +87,17 @@ pub fn clear_stw() { EVAL_BREAKER_WORD.fetch_and(!EB_STW, Ordering::Release); } +pub fn set_finalizing() { + EVAL_BREAKER_WORD.fetch_or(EB_FINALIZING, Ordering::Release); +} + +pub fn set_gc_interp() { + EVAL_BREAKER_WORD.fetch_or(EB_GC_INTERP, Ordering::Release); +} + /// Every flag must fit in the word the poll actually loads. Checked per target, /// so a flag too wide for a 32-bit `usize` fails the wasm32 build rather than /// silently reading as unarmed there. -const _: () = assert!((EB_ASYNC | EB_STW) < (1 << (EVAL_BREAKER_WORD_SIZE * 8 - 1))); +const _: () = assert!( + (EB_ASYNC | EB_STW | EB_FINALIZING | EB_GC_INTERP) < (1 << (EVAL_BREAKER_WORD_SIZE * 8 - 1)) +); diff --git a/majit/majit-metainterp/src/compile.rs b/majit/majit-metainterp/src/compile.rs index 97255c1a3fe..6bd4becc2a3 100644 --- a/majit/majit-metainterp/src/compile.rs +++ b/majit/majit-metainterp/src/compile.rs @@ -434,6 +434,15 @@ pub(crate) fn build_guard_metadata>( .map_or(false, |d| d.is_resume_guard() || d.is_resume_guard_copied()) { fd.set_fail_index_per_trace(fail_index); + // `ops` is the optimized frontend trace retained by + // `CompiledTrace`. The backend GC rewriter inserts + // operations before code generation, so the assembler's + // prepared-op index is not a valid index into this slice. + // Re-stamp the canonical descr from the frontend op, + // matching RPython where the ResumeGuardDescr remains + // attached to the live ResOperation rather than carrying + // an index into a separate rewritten array. + fd.set_source_op_index(op_idx); } } } @@ -2712,6 +2721,33 @@ mod tests { ); } + #[test] + fn test_build_guard_metadata_restamps_frontend_source_op_index() { + let inputargs = vec![InputArg::new_int(0)]; + let value = rooted_inputarg_operand(Type::Int, 0); + let prefix = Op::new(OpCode::SameAsI, std::slice::from_ref(&value)); + let descr = make_fail_descr_with_index(0, 1); + let fd = descr.as_fail_descr().unwrap(); + // The GC-rewritten backend trace can insert operations before a + // guard and temporarily stamp its prepared-op index on the shared + // descr. That index is not valid in the retained frontend trace. + fd.set_source_op_index(99); + let mut guard = Op::with_descr( + OpCode::GuardTrue, + std::slice::from_ref(&value), + descr.clone(), + ); + guard.setfailargs(smallvec::smallvec![value]); + guard.set_fail_arg_types(vec![Type::Int]); + + let (_resume_data, exit_layouts) = + build_guard_metadata(&inputargs, &[prefix, guard], 0, None); + let exit = exit_layouts.get(&0).expect("guard exit layout"); + + assert_eq!(exit.source_op_index, Some(1)); + assert_eq!(fd.source_op_index(), Some(1)); + } + #[test] fn test_patch_new_loop_reemits_ops_through_forwarded_results() { let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0); diff --git a/majit/majit-translate/src/codegen.rs b/majit/majit-translate/src/codegen.rs index ab9914e0270..0971ce400c9 100644 --- a/majit/majit-translate/src/codegen.rs +++ b/majit/majit-translate/src/codegen.rs @@ -172,6 +172,7 @@ mod tests { }], jitcodes: vec![main_jitcode], jitcodes_by_path: indexmap::IndexMap::new(), + indirectcalltarget_indices: Vec::new(), insns: indexmap::IndexMap::new(), descrs: Vec::new(), all_liveness: Vec::new(), diff --git a/majit/majit-translate/src/codewriter/call.rs b/majit/majit-translate/src/codewriter/call.rs index e50e7b3bfed..298fec65083 100644 --- a/majit/majit-translate/src/codewriter/call.rs +++ b/majit/majit-translate/src/codewriter/call.rs @@ -570,7 +570,13 @@ impl GraphStore { /// signature stays that of the shared graph, which the aliases resolve /// to anyway. pub(crate) fn insert(&mut self, path: CallPath, graph: FunctionGraph) { - let key = (graph.owner_root.clone(), graph.name.clone()); + let key = ( + graph + .source_identity + .clone() + .or_else(|| graph.owner_root.clone()), + graph.name.clone(), + ); match self.graphs.get_mut(&key) { Some(existing) => { existing.graph.func.merge_from(&graph.func); @@ -3111,6 +3117,17 @@ impl CallControl { for path in &todo { self.candidate_graphs.insert(path.clone()); } + // PyPy's portal reaches `BuiltinCode.funcrun`, whose `self.func` PBC + // contributes every gateway body to the indirect-call candidate set. + // Pyre's opcode walker lowers the equivalent Python CALL directly to + // `bh_call_fn`, bypassing that source-level dispatch graph, so seed the + // same generated-wrapper PBC family explicitly. This is the builtin + // gateway analogue of call.py:59-64's oopspec helper seeds below. + for path in self.builtin_wrapper_indirect_graphs() { + if self.candidate_graphs.insert(path.clone()) { + todo.push(path); + } + } // call.py:59-64 — seed the BFS with builtin oopspec helpers so // `int_abs` / `int_floordiv` / `int_mod` / `ll_math.ll_math_sqrt` // are reachable even when the portal does not call them @@ -3252,6 +3269,9 @@ impl CallControl { // `c_graphs` family, `None` meaning "unknown // family" and classifying the site as residual. OpKind::IndirectCall { graphs, .. } => match graphs { + Some(graphs) if graphs.is_empty() => { + self.builtin_wrapper_indirect_graphs() + } Some(graphs) => graphs.clone(), None => continue, }, @@ -3674,6 +3694,16 @@ impl CallControl { let arc = self.get_jitcode(&portal); self.jitdrivers_sd[jd_index].mainjitcode = Some(arc); } + // RPython reaches `BuiltinCode.func` as an indirect SomePBC call + // while transforming the portal closure; handling that call invokes + // `get_jitcode()` for each candidate graph. Pyre's opcode walker + // emits `bh_call_fn` directly and therefore has no source-level + // indirect op to perform the allocation. Materialise the same PBC + // family here so runtime fnaddr dispatch can resolve each generated + // gateway body to its JitCode. + for wrapper in self.builtin_wrapper_indirect_graphs() { + self.get_jitcode(&wrapper); + } } /// RPython: `CallControl.enum_pending_graphs()` (call.py:150-153). @@ -4505,6 +4535,35 @@ impl CallControl { .collect() } + /// Candidate PBC family for the generated `BuiltinCode.func` + /// function-pointer field. + /// + /// RPython obtains this list from the annotator's `SomePBC` + /// descriptions. Pyre's generated wrappers publish real fnaddrs through + /// `jit_trace_fnaddrs`; pair those addresses with their registered source + /// graphs here. Aliases sharing an address name the same wrapper; select + /// the most-qualified source identity for the one graph object entered + /// into the PBC family. + pub fn builtin_wrapper_indirect_graphs(&self) -> Vec { + let mut by_address: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for (path, &fnaddr) in &self.function_fnaddrs { + let Some(leaf) = path.last_segment() else { + continue; + }; + if !leaf.starts_with("__pyre_wrap_") || !self.function_graphs.contains_key(path) { + continue; + } + by_address.entry(fnaddr).or_default().push(path.clone()); + } + let mut result = Vec::new(); + for mut aliases in by_address.into_values() { + aliases.sort_by_key(|path| std::cmp::Reverse(path.segments.len())); + result.push(aliases.remove(0)); + } + result + } + /// RPython `call.py:259-280` — family-wide validation for indirect_call. /// /// Rejects a family if any member is marked `_elidable_function_` / diff --git a/majit/majit-translate/src/codewriter/jtransform.rs b/majit/majit-translate/src/codewriter/jtransform.rs index d7063a53d92..f77fed33319 100644 --- a/majit/majit-translate/src/codewriter/jtransform.rs +++ b/majit/majit-translate/src/codewriter/jtransform.rs @@ -2826,6 +2826,31 @@ impl<'a> Transformer<'a> { kind: OpKind::ConstRefNull, }]); } + // `rbuiltin.py:412-418 rtype_const_result` / + // `translator/rtyper/rbuiltin.rs::rtype_ptr_null`: by the time + // jtransform runs, `ptr::null[_mut]()` is a typed null pointer + // constant, not a residual host call. Pyre's rtyper currently types an + // ephemeral oracle rather than rewriting the surviving model graph, + // so apply that literal rewrite here. This is the null half of the + // niche `Option>` / `Option<&T>` representation emitted by + // `front::mir`; leaving it as a call would bake an unregistered + // symbolic fnaddr into every generated nullity test. + if let CallTarget::FunctionPath { segments } = target + && args.is_empty() + && matches!(result_ty, ValueType::Ref(_)) + && matches!( + segments.as_slice(), + [owner, ptr, leaf] + if matches!(owner.as_str(), "core" | "std") + && ptr == "ptr" + && matches!(leaf.as_str(), "null" | "null_mut") + ) + { + return RewriteResult::Replace(vec![SpaceOperation { + result: op.result.clone(), + kind: OpKind::ConstRefNull, + }]); + } // `rewrite_op_cast_pointer` → `rewrite_op_same_as` // (jtransform.py:254-257): the JIT does not distinguish a // down-cast pointer from its source, so the @@ -8671,6 +8696,36 @@ mod tests { )); } + #[test] + fn ptr_null_builtin_rewrites_to_null_ref_constant() { + let config = GraphTransformConfig::default(); + let mut graph = FunctionGraph::new("ptr_null_constant"); + let entry = graph.startblock; + let result_var = graph + .push_op_var( + entry, + OpKind::Call { + target: CallTarget::function_path(["core", "ptr", "null_mut"]), + args: vec![], + result_ty: ValueType::Ref(None), + }, + true, + ) + .unwrap(); + FunctionGraph::set_concretetype_of_inline(&result_var, ConcreteType::GcRef); + graph.set_return(entry, Some(result_var.clone())); + + let result = transform_graph(&graph, &config); + let folded = result + .graph + .blocks + .iter() + .flat_map(|block| &block.operations) + .find(|op| op.result.as_ref() == Some(&result_var)) + .expect("null result must survive as a constant definition"); + assert!(matches!(folded.kind, OpKind::ConstRefNull)); + } + /// PyPy parity regression guard: the qualified spellings /// `Result::Ok`, `Option::Some`, `std::result::Result::Err` etc. /// must elide identically to the bare `Ok` / `Some` / `Err` diff --git a/majit/majit-translate/src/front/mir.rs b/majit/majit-translate/src/front/mir.rs index d00ee26b292..03a613b69d7 100644 --- a/majit/majit-translate/src/front/mir.rs +++ b/majit/majit-translate/src/front/mir.rs @@ -62,10 +62,10 @@ //! - `Goto { target }` — direct edge. //! - `Switch { discr, targets }` — `ExitSwitch::Value` + per-arm //! `Link` with `ExitCase::Bool` / `ExitCase::Const`. -//! - `Call` — Direct / Trait → `Call(FunctionPath)`; Dynamic → -//! synthetic `Call(__dyn_call)` threading the fat-pointer -//! receiver. (A faithful `IndirectCall` lowering needs vtable -//! metadata Charon does not yet surface.) +//! - `Call` — Direct / Trait → `Call(FunctionPath)`; Dynamic vtable calls +//! use the trait-family indirect pipeline, plain function pointers become +//! `IndirectCall`, and the remaining closure shims use synthetic +//! `Call(__dyn_call)`. //! - `Drop` — pass-through `Goto` (JIT does not model destructor //! semantics). //! - `Assert` — strip and forward to the success target. @@ -924,6 +924,13 @@ fn build_semantic_program_from_llbc_with_static_addrs_filtered( // like a free function to the canonical registration loop and // the impl-key return-type / hint registrations get dropped. let self_ty_root = impl_method_owner_for_fundecl(llbc, fd).map(|(owner, _)| owner); + let graph = if let Some(owner) = &self_ty_root { + graph + .with_owner_root(owner.clone()) + .with_source_identity(format!("{module_path}::{owner}::{name}")) + } else { + graph.with_source_identity(fn_path.clone()) + }; // Surface trait identity for trait-impl methods so the // canonical registration loop can call `register_trait_method` // instead of routing through `extract_trait_impls`. Inherent @@ -3634,6 +3641,12 @@ impl<'a> Lowering<'a> { PlaceKind::Local(i) => Some(*i as usize), _ => None, }; + // RPython writeanalyze keys an array effect by the base box's + // concrete ARRAY type (`op.args[0].concretetype`). Charon keeps the + // same owner on the pre-projection Place, so preserve it before + // consuming `inner`. + let (projection_array_type_id, projection_array_nolength) = + array_projection_metadata(&inner.ty, self.llbc); let base = self.resolve_place(mir_bb, inner)?; let bb_id = self.block_id[mir_bb]; let op = match &elem { @@ -3723,9 +3736,9 @@ impl<'a> Lowering<'a> { base, index: idx_var, value: value.clone(), - item_ty: ValueType::Int, - array_type_id: None, - nolength: false, + item_ty: tyref_to_value_type(dest_ty, self.llbc), + array_type_id: projection_array_type_id, + nolength: projection_array_nolength, } } else { return Err(LowerError::Unsupported(format!( @@ -4792,6 +4805,7 @@ impl<'a> Lowering<'a> { && let Some(index_payload) = v.as_object().and_then(|m| m.get("Index")) { let idx_var = self.index_offset_var(mir_bb, index_payload)?; + let (array_type_id, nolength) = array_projection_metadata(&inner.ty, self.llbc); let base = self.resolve_place(mir_bb, *inner)?; let bb_id = self.block_id[mir_bb]; let res = self @@ -4803,8 +4817,8 @@ impl<'a> Lowering<'a> { base, index: idx_var, item_ty: tyref_to_value_type(&place_ty, self.llbc), - array_type_id: None, - nolength: false, + array_type_id, + nolength, pure: false, }, }); @@ -6011,6 +6025,12 @@ impl<'a> Lowering<'a> { tyref_node(&call.dest.ty, self.llbc) .and_then(|n| strip_ty_wrappers(n, self.llbc)) .and_then(|n| raw_ptr_pointee_class_root(n, self.llbc)) + // `Option<&mut RegisteredStruct>` is a nullable pointer + // niche, not a boxed Option object. RPython represents it as + // `SomeInstance(Struct, can_be_None=True)`: narrow to the + // payload class directly so a generated gateway wrapper's + // successful match arm can dispatch `self.method()`. + .or_else(|| self.option_ref_payload_class_root(&call.dest.ty)) // A `dont_look_inside` residual returning `Option<*mut PyObject>` // erases the same way — `dont_look_inside_return_token` maps it to // the `ref` GCREF token, so `result_ty` is `Ref(None)` too — but its @@ -6297,7 +6317,12 @@ impl<'a> Lowering<'a> { kind: OpKind::ArrayRead { base: args[0].clone(), index: args[1].clone(), - item_ty: ValueType::Ref(None), + // `Index::index(_mut)` returns `&T`/`&mut T`, + // while RPython's getarrayitem returns `T`. + // Preserve that pointee kind: treating the + // reference wrapper itself as the item makes an + // integer Vec load flow into `int_mul/ri>i`. + item_ty: tyref_deref_value_type(&call.dest.ty, self.llbc), array_type_id: None, nolength: false, pure: false, @@ -6695,7 +6720,70 @@ impl<'a> Lowering<'a> { self.graph.set_goto(bb_id, target_bb, link_args); return Ok(()); } - // `<[T]>::len` / `Vec::len` returns the container element + // `<[T]>::is_empty` is `arraylen_gc(s) == 0`. Keep both + // operations in the graph instead of residualizing the + // graph-less std helper. + if args.len() == 1 && self.is_slice_is_empty(®) { + let len = self + .graph + .alloc_value_var_with_type(crate::model::ConcreteType::Unknown); + self.graph.block_mut(bb_id).operations.push(SpaceOperation { + result: Some(len.clone()), + kind: OpKind::ArrayLen { + base: args[0].clone(), + array_type_id: None, + nolength: false, + }, + }); + let zero = self + .graph + .alloc_value_var_with_type(crate::model::ConcreteType::Unknown); + self.graph.block_mut(bb_id).operations.push(SpaceOperation { + result: Some(zero.clone()), + kind: OpKind::ConstInt(0), + }); + let res = self + .graph + .alloc_value_var_with_type(crate::model::ConcreteType::Unknown); + self.graph.block_mut(bb_id).operations.push(SpaceOperation { + result: Some(res.clone()), + kind: OpKind::BinOp { + op: "eq".to_string(), + lhs: len, + rhs: zero, + result_ty: ValueType::Int, + }, + }); + self.local_var[dest_local] = Some(res); + let target_bb = self.block_id[target]; + let link_args = self.edge_args(mir_bb, target)?; + self.graph.set_goto(bb_id, target_bb, link_args); + return Ok(()); + } + // `<[T]>::len` is already a low-level GcArray operation in + // the translated type model. Emit ArrayLen directly, just + // as `Rvalue::Len(place)` eventually does, so a gateway + // wrapper's red `&[PyObjectRef]` argument never detours + // through an unregistered host residual. + if args.len() == 1 && self.is_slice_len(®) { + let res = self + .graph + .alloc_value_var_with_type(crate::model::ConcreteType::Unknown); + self.graph.block_mut(bb_id).operations.push(SpaceOperation { + result: Some(res.clone()), + kind: OpKind::ArrayLen { + base: args[0].clone(), + array_type_id: None, + nolength: false, + }, + }); + self.local_var[dest_local] = Some(res); + let target_bb = self.block_id[target]; + let link_args = self.edge_args(mir_bb, target)?; + self.graph.set_goto(bb_id, target_bb, link_args); + return Ok(()); + } + // `Vec::len` returns the container element // count. Emit the `__len` operation on the receiver — the // rtyper routes it through the `len` op // (`flowspace_adapter`), which on a `SomeList` receiver @@ -7198,6 +7286,7 @@ impl<'a> Lowering<'a> { // fn-ptr / `FnOnce`, not a vtable slot; it keeps the // synthetic `__dyn_call` path (the fat-pointer receiver // threaded into `args[0]`). + let is_fn_ptr = operand_is_fn_ptr(&dyn_operand, self.llbc); let indirect = dyn_indirect_enabled() .then(|| self.dyn_indirect_target(&dyn_operand)) .flatten(); @@ -7207,6 +7296,22 @@ impl<'a> Lowering<'a> { args, result_ty, } + } else if is_fn_ptr { + // RPython `BuiltinCode.func` (and any other plain ll + // function-pointer field) is a PBC. Its rtyped call is + // `indirect_call(funcptr, *args, c_graphs)`, not a + // synthetic opaque helper. `Some([])` is the + // pre-CallControl marker for the generated builtin + // wrapper family; `rpbc::lower_indirect_calls` fills the + // candidate list once all registered graphs/fnaddrs are + // available. + let funcptr = self.resolve_operand(mir_bb, dyn_operand)?; + OpKind::IndirectCall { + funcptr, + args, + graphs: Some(Vec::new()), + result_ty, + } } else { let recv = self.resolve_operand(mir_bb, dyn_operand)?; let mut full_args = Vec::with_capacity(args.len() + 1); @@ -8943,8 +9048,7 @@ impl<'a> Lowering<'a> { self.llbc.fn_by_id(*id).is_some_and(|fd| { matches!( fd.item_meta.name_path().as_str(), - "core::slice::::len" - | "alloc::vec::::len" + "alloc::vec::::len" | "pyre_object::object_array::::len" | "pyre_object::int_array::::len" | "pyre_object::float_array::::len" @@ -8952,6 +9056,24 @@ impl<'a> Lowering<'a> { }) } + fn is_slice_len(&self, reg: &RegularCall) -> bool { + let CallKind::Fun(FunId::Regular { id }) = ®.kind else { + return false; + }; + self.llbc + .fn_by_id(*id) + .is_some_and(|fd| fd.item_meta.name_path() == "core::slice::::len") + } + + fn is_slice_is_empty(&self, reg: &RegularCall) -> bool { + let CallKind::Fun(FunId::Regular { id }) = ®.kind else { + return false; + }; + self.llbc + .fn_by_id(*id) + .is_some_and(|fd| fd.item_meta.name_path() == "core::slice::::is_empty") + } + /// `Vec::as_slice` / `<[T]>::as_slice` — a borrowed slice view of the /// same elements over shared storage, so it is an identity on the list /// model and the callsite aliases its receiver instead of leaving the @@ -9683,6 +9805,40 @@ impl<'a> Lowering<'a> { )) } + /// Registered ADT pointee of `Option<&T>` / `Option<&mut T>`. + /// + /// Rust uses the null pointer niche for this Option shape. It therefore + /// maps to RPython's nullable `SomeInstance(T)`, not to an Option + /// container class with `__discriminant` / `__pos_0` fields. + fn option_ref_payload_class_root(&self, dest_ty: &TyRef) -> Option { + if !crate::front::result_exc::tyref_is_option(dest_ty, self.llbc) { + return None; + } + let mut payload = tyref_node(dest_ty, self.llbc)? + .as_object()? + .get("Adt")? + .get("generics")? + .get("types")? + .get(0)?; + loop { + let obj = payload.as_object()?; + if let Some(id) = obj.get("Deduplicated").and_then(serde_json::Value::as_u64) { + payload = self.llbc.dedup_body(id)?; + continue; + } + if let Some(parts) = obj + .get("HashConsedValue") + .and_then(serde_json::Value::as_array) + && parts.len() == 2 + { + payload = &parts[1]; + continue; + } + let pointee = obj.get("Ref")?.as_array()?.get(1)?; + return adt_node_class_root(strip_ty_wrappers(pointee, self.llbc)?, self.llbc); + } + } + /// Project a raw Charon type node (a `generics.types` entry) to a /// [`ValueType`], first peeling the `HashConsedValue` / `Deduplicated` /// wrappers the entry may carry so [`tyref_to_value_type`]'s primitive @@ -10066,22 +10222,22 @@ impl<'a> Lowering<'a> { } } - /// `true` when `ty` is a niche-optimised `Option>` — an - /// `Option` whose sole payload is a `core::ptr::non_null::NonNull` - /// wrapper. Rust encodes such an Option in ONE pointer word (`None` - /// = null, `Some(p)` = the non-null pointer), so `Discriminant` on it - /// is a pointer-null test (`base != null`) and the `Some` payload read - /// is the identity on that pointer — no aggregate `__discriminant` / - /// `__pos_0` field exists. This mirrors the fieldless-enum by-value - /// model ([`Self::tyref_is_fieldless_enum`]). + /// `true` when `ty` is a niche-optimised `Option>` or + /// `Option<&mut T>`. Rust encodes each in ONE pointer + /// word (`None` = null, `Some(p)` = the non-null pointer), so + /// `Discriminant` on it is a pointer-null test (`base != null`) and the + /// `Some` payload read is the identity on that pointer — no aggregate + /// `__discriminant` / `__pos_0` field exists. This mirrors the + /// fieldless-enum by-value model ([`Self::tyref_is_fieldless_enum`]). /// /// Gated strictly to the `NonNull` payload: a raw `*mut T` / `*const T` /// payload (`Option<*mut PyObject>`) has NO null niche — Rust lays it /// out as a two-word tagged aggregate (discriminant word + pointer /// word), so folding it to a one-word pointer would read the tag word - /// as the payload. A `&T` payload is also niche, but the object - /// pointers this models are spelled `NonNull`, so only that shape - /// matches. + /// as the payload. Mutable references are included explicitly: Rust + /// guarantees their non-null representation, and generated + /// `#[pyre_class]::from_obj` returns `Option<&mut Self>`. Shared + /// references are not — see [`type_node_is_mut_ref`]. fn tyref_is_niche_option_ptr(&self, ty: &TyRef) -> bool { if !crate::front::result_exc::tyref_is_option(ty, self.llbc) { return false; @@ -10096,10 +10252,15 @@ impl<'a> Lowering<'a> { .and_then(|g| g.get("types")) .and_then(|t| t.as_array()) .and_then(|t| t.first()) - .and_then(|p| strip_ty_wrappers(p, self.llbc)) else { return false; }; + if type_node_is_mut_ref(payload, self.llbc) { + return true; + } + let Some(payload) = strip_ty_wrappers(payload, self.llbc) else { + return false; + }; let Some(def_id) = adt_node_def_id(payload) else { return false; }; @@ -11527,6 +11688,53 @@ impl<'a> Lowering<'a> { } } +fn operand_is_fn_ptr(operand: &Operand, llbc: &Llbc) -> bool { + let ty = match operand { + Operand::Copy(place) | Operand::Move(place) => &place.ty, + Operand::Const(_) => return false, + }; + let Some(fnptr) = tyref_node(ty, llbc) + .and_then(|node| strip_ty_wrappers(node, llbc)) + .and_then(|node| node.get("FnPtr")) + .and_then(serde_json::Value::as_object) + else { + return false; + }; + let Some(signature) = fnptr + .get("skip_binder") + .and_then(serde_json::Value::as_object) + else { + return false; + }; + if signature + .get("is_unsafe") + .and_then(serde_json::Value::as_bool) + != Some(false) + { + return false; + } + let Some(inputs) = signature + .get("inputs") + .and_then(serde_json::Value::as_array) + .filter(|inputs| inputs.len() == 1) + else { + return false; + }; + let input = charon_type_value_to_ast_string(&inputs[0], llbc, 0); + let output = signature + .get("output") + .map(|value| charon_type_value_to_ast_string(value, llbc, 0)) + .unwrap_or_default(); + // Exact source signature of `gateway::BuiltinCodeFn`: + // `fn(&[PyObjectRef]) -> Result`. Keep other + // one-argument callbacks in their existing residual `__dyn_call` family. + input.starts_with('[') + && input.contains("PyObject") + && output.starts_with("Result<") + && output.contains("PyObject") + && output.contains("PyError") +} + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -13279,6 +13487,19 @@ fn tyref_to_value_type(ty: &TyRef, llbc: &Llbc) -> ValueType { ValueType::Ref(None) } +/// Register-bank kind of the value behind a Charon reference destination. +/// +/// `Index::index` and `IndexMut::index_mut` expose `&T` / `&mut T`, but their +/// devirtualized flow operation is RPython's `getarrayitem`, whose result is +/// `T`. Preserve that distinction instead of banking the reference wrapper +/// itself as a GC ref. +fn tyref_deref_value_type(ty: &TyRef, llbc: &Llbc) -> ValueType { + let Some(node) = tyref_node(ty, llbc).and_then(|node| strip_ty_wrappers(node, llbc)) else { + return ValueType::Ref(None); + }; + tyref_to_value_type(&TyRef::Other(node.clone()), llbc) +} + /// Free-function form of [`Lowering::tyref_is_fieldless_enum`] for the /// standalone [`tyref_to_value_type`] helper (which holds no `Lowering`): /// `true` when `ty` resolves to an enum with at least one variant and @@ -13855,6 +14076,50 @@ fn cast_pointer_marker_op(root: String, arg: Variable) -> OpKind { } } +/// Whether a Charon type node's top-level constructor is a MUTABLE reference, +/// after following only serialization indirections. Unlike +/// [`strip_ty_wrappers`], this deliberately does not peel the reference +/// itself: callers use the result to distinguish Rust's guaranteed non-null +/// `&mut T` representation from nullable raw pointers. +/// +/// Shared references carry the same null niche, but are deliberately excluded. +/// `Option<&T>` is the `Iterator::next` result shape, and `front::iter_next` +/// rewrites its `__discriminant` match diamond into the `[__iter_next]` op — +/// folding that discriminant to a pointer null test leaves the rewrite with no +/// diamond to match, so the residual `Iterator::next()` call survives as the +/// unregistered callee the rewrite exists to remove. Teach `front::iter_next` +/// the null-test shape before widening this to shared references. +fn type_node_is_mut_ref<'l>(mut node: &'l serde_json::Value, llbc: &'l Llbc) -> bool { + for _ in 0..24 { + let Some(obj) = node.as_object() else { + return false; + }; + if let Some(id) = obj.get("Deduplicated").and_then(serde_json::Value::as_u64) { + let Some(body) = llbc.dedup_body(id) else { + return false; + }; + node = body; + continue; + } + if let Some(arr) = obj + .get("HashConsedValue") + .and_then(serde_json::Value::as_array) + && arr.len() == 2 + { + node = &arr[1]; + continue; + } + // `{"Ref": [region, ty, kind]}` — `kind` is `"Shared" | "Mut" | …`. + return obj + .get("Ref") + .and_then(serde_json::Value::as_array) + .and_then(|arr| arr.get(2)) + .and_then(serde_json::Value::as_str) + .is_some_and(|kind| kind.to_ascii_lowercase().contains("mut")); + } + false +} + /// Strip the indirection wrappers a Charon type node can carry — /// `{"Deduplicated": id}` / `{"HashConsedValue": [id, ty]}` / /// `{"Ref": [region, ty, kind]}` — and return the underlying type node @@ -14260,6 +14525,19 @@ fn tyref_to_ast_string(ty: &TyRef, llbc: &Llbc) -> String { } } +/// Concrete ARRAY metadata for a Charon `ProjectionElem::Index`. +/// +/// The identity stays on the Place itself, matching RPython's +/// `box.concretetype`; no identity-keyed side table is introduced. +fn array_projection_metadata(ty: &TyRef, llbc: &Llbc) -> (Option, bool) { + let identity = tyref_to_ast_string(ty, llbc); + if identity.starts_with("??") { + return (None, false); + } + let nolength = crate::front::typestr::nolength_from_array_type_id(Some(identity.as_str())); + (Some(identity), nolength) +} + /// Recursive worker for [`tyref_to_ast_string`] operating on a raw /// Charon type-expression `Value` (a TyRef body or a nested /// generic-argument type). `depth` guards against pathological cycles. @@ -14955,6 +15233,31 @@ fn charon_const_generic_to_string(cg: &serde_json::Value) -> String { return s.to_string(); } if let Some(obj) = cg.as_object() { + // Current Charon `ConstGeneric::Value` schema: + // + // {"kind":{"Literal":{"Scalar":{"Unsigned":["Usize","624"]}}}, + // "ty": ...} + // + // Preserve the literal's decimal spelling. This is the concrete N + // in Rust `[T; N]`, and therefore part of the exact ARRAY identity + // RPython would keep in `op.args[0].concretetype`. + if let Some(scalar) = obj + .get("kind") + .and_then(|v| v.get("Literal")) + .and_then(|v| v.get("Scalar")) + .and_then(serde_json::Value::as_object) + && let Some(n) = scalar.values().find_map(|v| { + v.as_array().and_then(|parts| parts.last()).and_then(|n| { + n.as_str() + .map(str::to_string) + .or_else(|| n.as_u64().map(|n| n.to_string())) + }) + }) + { + return n; + } + // Older serialized Charon schema retained for frozen LLBC + // compatibility. if let Some(val) = obj.get("Value") { if let Some(scalar) = val .as_object() @@ -18056,7 +18359,7 @@ fn collapse_panic_message_chains(graph: &mut FunctionGraph) -> usize { mod tests { use super::harden_duplicate_leaf_metadata; use super::{ - DecodedConst, cast_kind_is_raw_ptr, cast_pointer_marker_op, + DecodedConst, cast_kind_is_raw_ptr, cast_pointer_marker_op, charon_const_generic_to_string, charon_type_value_to_ast_string, decode_literal, }; use majit_charon_reader::Llbc; @@ -18084,6 +18387,120 @@ mod tests { )); } + #[test] + fn current_charon_const_generic_scalar_preserves_array_length() { + let value = serde_json::json!({ + "kind": { + "Literal": { + "Scalar": { + "Unsigned": ["Usize", "624"] + } + } + }, + "ty": { + "Literal": "Usize" + } + }); + assert_eq!(charon_const_generic_to_string(&value), "624"); + } + + /// Loads the real interpreter LLBC to anchor fixed-array effect identity + /// to `_random::Random::genrand32`. + #[test] + #[ignore] + fn random_genrand32_fixed_array_keeps_concrete_identity() { + use crate::model::{OpKind, ValueType}; + + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../build/llbc/pyre-interpreter.ullbc" + ); + let llbc = Llbc::load(path).expect("load real LLBC"); + let fd = llbc + .iter_local_fns() + .find(|fd| fd.item_meta.name_path().ends_with("::genrand32")) + .expect("_random::Random::genrand32 in interpreter LLBC"); + let graph = super::lower_fun_decl(&llbc, fd).expect("lower Random::genrand32"); + + let mut reads = 0usize; + let mut writes = 0usize; + for op in graph.blocks.iter().flat_map(|block| &block.operations) { + match &op.kind { + OpKind::ArrayRead { + item_ty, + array_type_id, + nolength, + .. + } => { + reads += 1; + assert_eq!(*item_ty, ValueType::Unsigned); + assert_eq!(array_type_id.as_deref(), Some("[u32;624]")); + assert!(*nolength); + } + OpKind::ArrayWrite { + item_ty, + array_type_id, + nolength, + .. + } => { + writes += 1; + assert_eq!(*item_ty, ValueType::Unsigned); + assert_eq!(array_type_id.as_deref(), Some("[u32;624]")); + assert!(*nolength); + } + _ => {} + } + } + assert!(reads >= 1); + assert!(writes >= 1); + } + + #[test] + #[ignore] + fn random_wrapper_narrows_nullable_self_to_w_random() { + use crate::model::{CallTarget, OpKind, ValueType}; + + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../build/llbc/pyre-interpreter.ullbc" + ); + let llbc = Llbc::load(path).expect("load real LLBC"); + let fd = llbc + .iter_local_fns() + .find(|fd| fd.item_meta.name_path().ends_with("::__pyre_wrap_random")) + .expect("_random::__pyre_wrap_random"); + let graph = super::lower_fun_decl(&llbc, fd).expect("lower wrapper"); + let ops: Vec<_> = graph + .blocks + .iter() + .flat_map(|block| &block.operations) + .collect(); + let receiver = ops + .iter() + .find_map(|op| match &op.kind { + OpKind::Call { + target: CallTarget::Method { name, .. }, + args, + .. + } if name == "random" => args.first().cloned(), + _ => None, + }) + .expect("random receiver"); + let producer = ops + .iter() + .find(|op| op.result.as_ref().is_some_and(|result| result == &receiver)) + .expect("random receiver producer"); + assert!(matches!( + &producer.kind, + OpKind::Call { + target: CallTarget::FunctionPath { segments }, + result_ty: ValueType::Ref(Some(root)), + .. + } if segments == &["__pyre_cast_instance".to_string(), "W_Random".to_string()] + && root == "W_Random" + )); + } + #[test] fn type_arg_splits_per_instantiation_defers_singlefloat_unit_and_empty() { use super::type_arg_splits_per_instantiation; diff --git a/majit/majit-translate/src/lib.rs b/majit/majit-translate/src/lib.rs index b3d71907f34..af15e8b8ce3 100644 --- a/majit/majit-translate/src/lib.rs +++ b/majit/majit-translate/src/lib.rs @@ -1748,6 +1748,7 @@ fn analyze_pipeline_from_module_paths( functions: Vec::new(), jit_drivers: Vec::new(), jitcodes: Vec::new(), + indirectcalltarget_indices: Vec::new(), jitcodes_by_path: indexmap::IndexMap::new(), insns: indexmap::IndexMap::new(), descrs: Vec::new(), @@ -1758,7 +1759,7 @@ fn analyze_pipeline_from_module_paths( }; mark_phase!("call_control + canonical_trait_impls + register graphs"); - let (jitcodes, insns, descrs, all_liveness) = + let (jitcodes, indirectcalltarget_indices, insns, descrs, all_liveness) = make_jitcodes(&config.pipeline, &mut call_control, &mut prof); mark_phase!("make_jitcodes"); pipeline.jit_drivers = call_control @@ -1774,6 +1775,7 @@ fn analyze_pipeline_from_module_paths( }) .collect(); pipeline.jitcodes = jitcodes; + pipeline.indirectcalltarget_indices = indirectcalltarget_indices; // Mirror of `CallControl::jitcodes` (RPython `call.py:87 self.jitcodes`) // captured before `call_control` is dropped. Needed because consumers // that look up a JitCode by graph identity cannot reconstruct the key @@ -1861,6 +1863,7 @@ fn make_jitcodes( prof: &mut PhaseProfiler, ) -> ( Vec>, + Vec, indexmap::IndexMap, Vec, Vec, @@ -1903,6 +1906,24 @@ fn make_jitcodes( codewriter.drain_pending_graphs(call_control, &pipeline_config.transform); prof.mark(" drain_pending_graphs"); + // `BuiltinCode.func` is a SomePBC function-pointer field. The ordinary + // translated indirect-call op contributes this family through + // `IndirectCallTargets`; pyre's interpreter call boundary hides that op + // behind the runtime `call_fn` helper, so publish the annotator's same + // finite wrapper family on the shared Assembler explicitly. The handles + // are the exact `CallControl.jitcodes` objects materialized by + // `grab_initial_jitcodes`, preserving RPython object identity. + let builtin_wrapper_targets: Vec = call_control + .builtin_wrapper_indirect_graphs() + .into_iter() + .filter_map(|path| call_control.jitcodes().get(&path).cloned()) + .map(jitcode::JitCodeHandle::from) + .collect(); + codewriter + .assembler + .indirectcalltargets + .extend(builtin_wrapper_targets); + // RPython codewriter.py:85: self.assembler.finished(callinfocollection). codewriter .assembler @@ -1913,6 +1934,14 @@ fn make_jitcodes( // jitcode receives its dense index when appended, matching RPython // `make_jitcodes()`. let jitcodes = call_control.collect_jitcodes_in_alloc_order(); + let mut indirectcalltarget_indices: Vec = codewriter + .assembler + .indirectcalltargets + .iter() + .map(|target| target.index()) + .collect(); + indirectcalltarget_indices.sort_unstable(); + indirectcalltarget_indices.dedup(); // RPython codewriter.py + assembler.py: `Assembler.insns` grows as // `write_insn` encounters new keys. We snapshot the final table @@ -1931,7 +1960,13 @@ fn make_jitcodes( // can resolve the `BC_LIVE` offsets baked into `JitCode.code`. let all_liveness = codewriter.assembler.all_liveness().to_vec(); - (jitcodes, insns, descrs, all_liveness) + ( + jitcodes, + indirectcalltarget_indices, + insns, + descrs, + all_liveness, + ) } /// Generate tracing code directly from the canonical pipeline result. @@ -2016,7 +2051,7 @@ mod portal_driver_tests { let mut policy = policy::DefaultJitPolicy::new(); call_control.find_all_graphs(&mut policy); - let (jitcodes, _, _, _) = + let (jitcodes, _, _, _, _) = make_jitcodes(&config, &mut call_control, &mut PhaseProfiler::new()); assert_eq!(jitcodes.len(), 1); assert_eq!(jitcodes[0].index(), 0); @@ -2085,7 +2120,7 @@ mod portal_driver_tests { let mut policy = policy::DefaultJitPolicy::new(); call_control.find_all_graphs(&mut policy); - let (jitcodes, _, _, _) = + let (jitcodes, _, _, _, _) = make_jitcodes(&config, &mut call_control, &mut PhaseProfiler::new()); let merge = jitcodes[0] .body() diff --git a/majit/majit-translate/src/model.rs b/majit/majit-translate/src/model.rs index c8c120c133a..23d9cda5fb0 100644 --- a/majit/majit-translate/src/model.rs +++ b/majit/majit-translate/src/model.rs @@ -4355,6 +4355,16 @@ impl FuncEffects { #[derive(Debug, Clone, PartialEq, Eq)] pub struct FunctionGraph { pub name: String, + /// Stable identity of the source function object. + /// + /// RPython keys graph metadata by the Python function object's identity, + /// not by `FunctionGraph.name`. Charon can emit many distinct generated + /// gateway functions with the same leaf name in different modules, so the + /// qualified declaration path is carried here to keep those funcobjs + /// distinct while aliases of one declaration continue to share a graph. + /// Synthetic/test graphs leave this as `None` and retain the historical + /// `(owner_root, name)` identity fallback. + pub source_identity: Option, /// Impl-block self-type root for graphs produced from `impl { fn m(&self, ...) }`. /// Mirrors PyPy's `graph.func.im_class` access (the bound-method's class /// reference): RPython lifts `self` as `SomeInstance(getuniqueclassdef(im_class))` @@ -4465,6 +4475,7 @@ impl FunctionGraph { notes: Vec::new(), return_type: None, owner_root: None, + source_identity: None, hints: Vec::new(), func: FuncEffects::default(), } @@ -4501,6 +4512,11 @@ impl FunctionGraph { self } + pub fn with_source_identity(mut self, identity: impl Into) -> Self { + self.source_identity = Some(identity.into()); + self + } + /// Return the canonical exception block and its `(etype, evalue)` /// inputarg Variables. /// diff --git a/majit/majit-translate/src/pipeline.rs b/majit/majit-translate/src/pipeline.rs index df09d866890..4a62526c832 100644 --- a/majit/majit-translate/src/pipeline.rs +++ b/majit/majit-translate/src/pipeline.rs @@ -111,6 +111,10 @@ pub struct ProgramPipelineResult { /// `JitDriverStaticData.mainjitcode` or `IndirectCallTargets`) share /// identity with the values appearing here. pub jitcodes: Vec>, + /// RPython `Assembler.indirectcalltargets`, encoded by identity as dense + /// indices into `jitcodes`. + #[serde(default)] + pub indirectcalltarget_indices: Vec, /// RPython: `rpython/jit/codewriter/call.py:87 self.jitcodes` /// (graph-keyed dict). Pyre uses `CallPath` as graph identity at the /// module boundary. Paired with `jitcodes` (which mirrors @@ -196,6 +200,7 @@ mod tests { main_jitcode_index: 0, }], jitcodes: vec![Arc::new(JitCode::new("consts"))], + indirectcalltarget_indices: Vec::new(), jitcodes_by_path: indexmap::IndexMap::new(), insns: indexmap::IndexMap::new(), descrs: Vec::new(), diff --git a/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs b/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs index 90b3c93c070..d8aaee21ef9 100644 --- a/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs +++ b/majit/majit-translate/src/translator/rtyper/flowspace_adapter.rs @@ -853,6 +853,9 @@ pub(crate) fn op_canraise(kind: &OpKind) -> bool { // getitem / setitem -> `[IndexError, KeyError, Exception]` // (operation.py:727-730). OpKind::ArrayRead { .. } | OpKind::ArrayWrite { .. } => true, + // `len` is a pure flowspace operation; the rtyper routes it through + // the receiver repr (`rtype_len`) without an exception edge. + OpKind::ArrayLen { .. } => false, // `InteriorField*` unfolds in `translate_op` into a chained // `getitem(base, index)` followed by `getattr` / `setattr`, so it // carries the getitem's `[IndexError, KeyError, Exception]` @@ -1324,7 +1327,7 @@ pub fn translate_op( )]) } - // ─── ArrayRead / ArrayWrite ports ─── + // ─── ArrayRead / ArrayLen / ArrayWrite ports ─── // RPython `flowspace/operation.py: GetItem.opname = 'getitem'` // and `setitem`. The base[index] form maps directly to // `getitem(base, index)` / `setitem(base, index, value)`; the @@ -1342,6 +1345,11 @@ pub fn translate_op( result, )]) } + OpKind::ArrayLen { base, .. } => { + let base_hl = lookup_operand(value_map, base, op, "base")?; + let result = resolve_result_hlvalue(op, value_map)?; + Ok(vec![FlowspaceOp::new("len", vec![base_hl], result)]) + } OpKind::ArrayWrite { base, index, value, .. } => { @@ -2502,6 +2510,7 @@ fn opkind_variant_name(kind: &OpKind) -> &'static str { OpKind::FieldRead { .. } => "FieldRead", OpKind::FieldWrite { .. } => "FieldWrite", OpKind::ArrayRead { .. } => "ArrayRead", + OpKind::ArrayLen { .. } => "ArrayLen", OpKind::ArrayWrite { .. } => "ArrayWrite", OpKind::InteriorFieldRead { .. } => "InteriorFieldRead", OpKind::InteriorFieldWrite { .. } => "InteriorFieldWrite", @@ -5145,6 +5154,29 @@ mod tests { assert_eq!(lowered.args.len(), 2); } + #[test] + fn translate_op_array_len_lowers_to_len() { + let mut value_map: HashMap = HashMap::new(); + let mut graph = LegacyGraph::new("translate_op_fixture"); + let vars = mint_vars(&mut graph, 4); + value_map.insert(vars[1].clone(), Hlvalue::Variable(Variable::new())); + value_map.insert(vars[2].clone(), Hlvalue::Variable(Variable::new())); + let op = SpaceOperation { + result: Some(vars[2].clone()), + kind: OpKind::ArrayLen { + base: vars[1].clone(), + array_type_id: Some("[PyObjectRef]".to_string()), + nolength: false, + }, + }; + let translated = + translate_op(&op, &value_map, &empty_call_registry()).expect("ArrayLen arm must lower"); + assert_eq!(translated.len(), 1); + assert_eq!(translated[0].opname, "len"); + assert_eq!(translated[0].args.len(), 1); + assert!(!op_canraise(&op.kind)); + } + #[test] fn translate_op_array_write_lowers_to_setitem() { let mut value_map: HashMap = HashMap::new(); diff --git a/majit/majit-translate/src/translator/rtyper/rpbc.rs b/majit/majit-translate/src/translator/rtyper/rpbc.rs index 6ad9aa98ad6..1551eaa8862 100644 --- a/majit/majit-translate/src/translator/rtyper/rpbc.rs +++ b/majit/majit-translate/src/translator/rtyper/rpbc.rs @@ -324,6 +324,25 @@ pub(crate) fn select_call_family_row( /// RPython's `convert_to_concrete_llfn` materialises the funcptr but the /// eventual `indirect_call` still receives `self, ...` as ordinary args. pub fn lower_indirect_calls(graph: &mut JitFunctionGraph, call_control: &CallControl) { + // Generated gateway wrappers enter the MIR graph as a plain function- + // pointer `IndirectCall` with `Some([])` as a deferred PBC-family marker. + // At rtype time CallControl owns both the translated graphs and the + // linker-resolved wrapper addresses, so fill the same `c_graphs` list + // `FunctionReprBase.call()` appends in rpbc.py:216. + let builtin_wrappers = call_control.builtin_wrapper_indirect_graphs(); + for block in &mut graph.blocks { + for op in &mut block.operations { + if let OpKind::IndirectCall { + graphs: Some(graphs), + .. + } = &mut op.kind + && graphs.is_empty() + { + *graphs = builtin_wrappers.clone(); + } + } + } + // Collect the (block, op_index) sites first so the rewrite below // can mutate the graph without aliasing the borrow. let sites: Vec<(usize, usize)> = graph diff --git a/pyre/pyre-interpreter/src/builtins.rs b/pyre/pyre-interpreter/src/builtins.rs index 2ed49f166f9..f2ff63e0f06 100644 --- a/pyre/pyre-interpreter/src/builtins.rs +++ b/pyre/pyre-interpreter/src/builtins.rs @@ -3321,17 +3321,14 @@ pub fn builtin_abs(args: &[PyObjectRef]) -> Result /// kwargs through `Arguments::_match_signature` into named slots, this helper /// and the `__pyre_kw__` marker can be removed. pub(crate) fn split_builtin_kwargs(args: &[PyObjectRef]) -> (&[PyObjectRef], Option) { - if let Some(&last) = args.last() { + if !args.is_empty() { + let last = args[args.len() - 1]; // The marker dict stores an unforgeable sentinel under `__pyre_kw__` // (`call_with_kwargs`), so detection is by value identity. A dict // passed positionally that merely contains a `__pyre_kw__` string key // (`float({'__pyre_kw__': True})`) carries a different value and is a // value, not the marker, so it must not be stripped. - let is_marker = unsafe { - is_dict(last) - && pyre_object::w_dict_getitem_str(last, "__pyre_kw__") - .is_some_and(pyre_object::kw_marker::is_kw_marker_sentinel) - }; + let is_marker = (unsafe { is_dict(last) }) && builtin_kwargs_marker_dict(last); if is_marker { return (&args[..args.len() - 1], Some(last)); } @@ -3339,6 +3336,19 @@ pub(crate) fn split_builtin_kwargs(args: &[PyObjectRef]) -> (&[PyObjectRef], Opt (args, None) } +/// Cold dictionary-strategy half of the flat builtin-keyword ABI. +/// +/// The caller first proves `last` is a dict. Keeping the strategy dispatch +/// residual lets the trace-visible non-dict path reject the marker test +/// without entering `w_dict_getitem_str`'s dynamic strategy function. +#[majit_macros::dont_look_inside] +pub fn builtin_kwargs_marker_dict(last: PyObjectRef) -> bool { + unsafe { + pyre_object::w_dict_getitem_str(last, "__pyre_kw__") + .is_some_and(pyre_object::kw_marker::is_kw_marker_sentinel) + } +} + /// True when the kwargs dict from [`split_builtin_kwargs`] carries a real /// keyword (any entry other than the `__pyre_kw__` marker). An empty /// `**{}` therefore reports `false`. @@ -3450,11 +3460,11 @@ pub(crate) fn bind_pos_or_kw( /// `true` when the last argument is the `__pyre_kw__`-tagged dict the /// CALL_KW builtin dispatch appends — i.e. the call carried keywords. pub(crate) fn has_builtin_kwargs(args: &[PyObjectRef]) -> bool { - matches!(args.last(), Some(&last) if unsafe { - is_dict(last) - && pyre_object::w_dict_getitem_str(last, "__pyre_kw__") - .is_some_and(pyre_object::kw_marker::is_kw_marker_sentinel) - }) + if args.is_empty() { + return false; + } + let last = args[args.len() - 1]; + (unsafe { is_dict(last) }) && builtin_kwargs_marker_dict(last) } /// Resolve a single positional-or-keyword builtin argument: prefer the diff --git a/pyre/pyre-interpreter/src/call.rs b/pyre/pyre-interpreter/src/call.rs index 8cf713b0a97..c61239d7bf6 100644 --- a/pyre/pyre-interpreter/src/call.rs +++ b/pyre/pyre-interpreter/src/call.rs @@ -757,14 +757,32 @@ fn set_orig_class(result: PyObjectRef, alias: PyObjectRef) -> Result<(), crate:: } } -// `dont_look_inside`: a builtin is invoked through a runtime `BuiltinCodeFn` -// value (`func(args)`), a call through a fn-pointer the tracer has no lowering -// for (only static `CallPath`s lower). The builtin body is the residual -// boundary — the JIT residualizes the whole dispatch (signature-aware kwarg -// packing + the C-level call) instead of tracing into it, mirroring -// `cpu.bh_call_*`. This also keeps `builtin_code_get_signature`'s raw-ptr -// `as_ref` read out of any traced graph. +fn finish_builtin_code_positional( + current_code: PyObjectRef, + current_args: &[PyObjectRef], +) -> PyResult { + if let Some(sig) = unsafe { crate::builtin_code_get_signature(current_code) } { + // Every HOPELESS signature needs `_match_signature`, not only + // *args/**kwargs/kw-only shapes. A plain optional positional + // parameter also has HOPELESS fast arity; bypassing the binder let + // excess positionals reach the typed wrapper, which consumes its + // declared prefix and silently ignores the rest. + if unsafe { crate::builtin_code_get_fast_natural_arity(current_code) } == crate::HOPELESS { + let fname = unsafe { crate::builtin_code_name(current_code) }; + let bound = bind_kwargs_to_signature(sig, fname, current_args, &[])?; + return unsafe { crate::builtin_code_call(current_code, &bound) }; + } + } + unsafe { crate::builtin_code_call(current_code, current_args) } +} + #[majit_macros::dont_look_inside] +fn call_builtin_code_many_from_roots(root_base: usize, nargs: usize) -> PyResult { + let mut rooted = vec![pyre_object::PY_NULL; 1 + nargs]; + pyre_object::gc_roots::shadow_stack_copy_range(root_base, &mut rooted); + finish_builtin_code_positional(rooted[0], &rooted[1..]) +} + fn call_builtin_code_positional(code: PyObjectRef, args: &[PyObjectRef]) -> PyResult { // `gateway.py:824 BuiltinCode.funcrun` is translated with both its code // object and `Arguments.arguments_w` live across gateway dispatch. A @@ -778,29 +796,37 @@ fn call_builtin_code_positional(code: PyObjectRef, args: &[PyObjectRef]) -> PyRe for &arg in args { pyre_object::gc_roots::pin_root(arg); } - let current_code = || pyre_object::gc_roots::shadow_stack_get(root_base); - let current_args = || { - (0..args.len()) - .map(|i| pyre_object::gc_roots::shadow_stack_get(root_base + 1 + i)) - .collect::>() - }; - - if let Some(sig) = unsafe { crate::builtin_code_get_signature(current_code()) } { - // Every HOPELESS signature needs `_match_signature`, not only - // *args/**kwargs/kw-only shapes. A plain optional positional - // parameter also has HOPELESS fast arity; bypassing the binder let - // excess positionals reach the typed wrapper, which consumes its - // declared prefix and silently ignores the rest. - if unsafe { crate::builtin_code_get_fast_natural_arity(current_code()) } == crate::HOPELESS - { - let fname = unsafe { crate::builtin_code_name(current_code()) }; - let args = current_args(); - let bound = bind_kwargs_to_signature(sig, fname, &args, &[])?; - return unsafe { crate::builtin_code_call(current_code(), &bound) }; + // RPython's pop-roots reload produces ordinary live variables. Spell the + // common fixed-arity cases the same way so source translation sees no Rust + // array slicing/indexing helpers between the live roots and the gateway + // indirect call. The uncommon variadic case stays a residual helper. + let current_code = pyre_object::gc_roots::shadow_stack_get(root_base); + match args.len() { + 0 => finish_builtin_code_positional(current_code, &[]), + 1 => { + let a0 = pyre_object::gc_roots::shadow_stack_get(root_base + 1); + finish_builtin_code_positional(current_code, &[a0]) + } + 2 => { + let a0 = pyre_object::gc_roots::shadow_stack_get(root_base + 1); + let a1 = pyre_object::gc_roots::shadow_stack_get(root_base + 2); + finish_builtin_code_positional(current_code, &[a0, a1]) + } + 3 => { + let a0 = pyre_object::gc_roots::shadow_stack_get(root_base + 1); + let a1 = pyre_object::gc_roots::shadow_stack_get(root_base + 2); + let a2 = pyre_object::gc_roots::shadow_stack_get(root_base + 3); + finish_builtin_code_positional(current_code, &[a0, a1, a2]) + } + 4 => { + let a0 = pyre_object::gc_roots::shadow_stack_get(root_base + 1); + let a1 = pyre_object::gc_roots::shadow_stack_get(root_base + 2); + let a2 = pyre_object::gc_roots::shadow_stack_get(root_base + 3); + let a3 = pyre_object::gc_roots::shadow_stack_get(root_base + 4); + finish_builtin_code_positional(current_code, &[a0, a1, a2, a3]) } + nargs => call_builtin_code_many_from_roots(root_base, nargs), } - let args = current_args(); - unsafe { crate::builtin_code_call(current_code(), &args) } } /// Leaf execution mode for a user-function call reached through diff --git a/pyre/pyre-interpreter/src/eval.rs b/pyre/pyre-interpreter/src/eval.rs index f25714c62c7..385bad5933f 100644 --- a/pyre/pyre-interpreter/src/eval.rs +++ b/pyre/pyre-interpreter/src/eval.rs @@ -1792,6 +1792,12 @@ fn eval_loop(frame: &mut PyFrame) -> PyResult { // a nested `eval_loop_jit` running under this one observes depth > 1 and // skips collection. No-op when the flag is off. let _eval_activation = pyre_object::gc_interp::EvalActivationGuard::enter(); + if _eval_activation.armed() { + // Publish the process-stable configuration in the breaker word once + // per activation, before the first dispatch. Compiled back-edges mask + // this bit out. + majit_ir::eval_breaker_word::set_gc_interp(); + } let _current_frame_guard = if frame.execution_context.is_null() { install_current_frame(frame) } else { @@ -1801,7 +1807,14 @@ fn eval_loop(frame: &mut PyFrame) -> PyResult { let mut next_instr = frame.next_instr(); loop { - crate::module::thread::park_if_finalizing(); + // PyPy's ActionFlag is one process breaker. Keep pyre's free-threaded + // finalization and STW extensions on the same already-established + // breaker word, so the ordinary dispatch pays one relaxed load rather + // than polling two process-global atomics independently. + let dispatch_breaker = majit_ir::eval_breaker_word::load(); + if dispatch_breaker & majit_ir::eval_breaker_word::EB_FINALIZING != 0 { + crate::module::thread::park_if_finalizing(); + } // Interpreter-path GC safepoint (PYRE_GC_INTERP), mirroring the JIT // eval loop. Between opcodes the only live refs are in the frame, // reachable through the installed `current_frame` root walker; no @@ -1809,13 +1822,17 @@ fn eval_loop(frame: &mut PyFrame) -> PyResult { // the flag is on and enough interpreter objects have accumulated. // Without it, a JIT-off run reclaims interpreter-routed old-gen // allocations only at explicit `gc.collect`, so RSS grows unbounded. - pyre_object::gc_interp::safepoint(); + if dispatch_breaker & majit_ir::eval_breaker_word::EB_GC_INTERP != 0 { + pyre_object::gc_interp::safepoint(); + } // Free-threaded stop-the-world rendezvous. Worker threads deliberately // execute this plain evaluator (their JitDriver state is thread-owned), // so they must poll the same process breaker as compiled/JIT-warm // loops; otherwise a non-allocating Python loop can prevent collection // and fork/finalization STW forever. - majit_gc::gc_sync::safepoint_poll(); + if dispatch_breaker & majit_ir::eval_breaker_word::EB_STW != 0 { + majit_gc::gc_sync::safepoint_poll(); + } if next_instr >= code.instructions.len() { return Ok(w_none()); @@ -4239,7 +4256,12 @@ impl OpcodeStepExecutor for PyFrame { // CPython 3.12+ CALL: stack is [callable, null_or_self, arg0..argN-1]. // null_or_self is NULL for plain calls, `self` for method calls. fn call(&mut self, nargs: usize) -> Result<(), PyError> { - // baseobjspace.py:1240-1261 fast path: Function + no method binding + // baseobjspace.py:1243-1266 fast path: Function, including the + // CALL_METHOD form. callmethod.py:85-94 counts a non-null `self` as + // one extra argument while `dropvalues` remains the physical + // `[callable, null_or_self, explicit args...]` width. This is what + // lets the translated interpreter expose an ordinary `_flat_pycall` + // to the meta-tracer for `obj.method(...)`, just like PyPy. // // baseobjspace.py:1243 — skip fast path when profiling is active // and the function wraps a builtin code (c_call/c_return events). @@ -4250,15 +4272,41 @@ impl OpcodeStepExecutor for PyFrame { // items above stack_base (callable + null_or_self + args). let stack_items = self.valuestackdepth.saturating_sub(self.stack_base()); if stack_items >= nargs + 2 && !self.get_is_being_profiled() { - let null_or_self = self.peekvalue_maybe_none(nargs); - let callable = self.peekvalue_maybe_none(nargs + 1); - if null_or_self.is_null() - && !callable.is_null() - && unsafe { crate::is_function(callable) } + let mut null_or_self = self.peekvalue_maybe_none(nargs); + let mut callable = self.peekvalue_maybe_none(nargs + 1); + // baseobjspace.py:1254-1259: `_Method` is not a generic callable + // here. Reuse its null/self stack slot for `w_instance`, unwrap + // `w_function`, and continue through the identical Function + // valuestack path. Module aliases such as `random.gauss = + // _inst.gauss` depend on this just as direct `obj.method()` calls + // do; allocating an Arguments Vec for every alias call diverges + // from PyPy's meta-traced interpreter shape. + if !callable.is_null() + && null_or_self.is_null() + && unsafe { pyre_object::is_method(callable) } { + let receiver = unsafe { pyre_object::w_method_get_self(callable) }; + let function = unsafe { pyre_object::w_method_get_func(callable) }; + if !receiver.is_null() + && !function.is_null() + && unsafe { crate::is_function(function) } + { + self.settopvalue(receiver, nargs); + null_or_self = receiver; + callable = function; + } + } + if !callable.is_null() && unsafe { crate::is_function(callable) } { + let methodcall = !null_or_self.is_null(); + let call_nargs = nargs + usize::from(methodcall); let anchor = FrameAnchor::new(self); - let result = - crate::function::funccall_valuestack(callable, nargs, self, nargs + 2, false); + let result = crate::function::funccall_valuestack( + callable, + call_nargs, + self, + nargs + 2, + methodcall, + ); if result.is_null() { return Err(crate::call::take_call_error() .unwrap_or_else(|| crate::PyError::type_error("call failed")) diff --git a/pyre/pyre-interpreter/src/executioncontext.rs b/pyre/pyre-interpreter/src/executioncontext.rs index e9ba891d1da..b913e1af857 100644 --- a/pyre/pyre-interpreter/src/executioncontext.rs +++ b/pyre/pyre-interpreter/src/executioncontext.rs @@ -646,12 +646,15 @@ impl ExecutionContext { Ok(()) } + #[inline(always)] pub fn bytecode_trace( &mut self, frame: *mut PyFrame, decr_by: usize, ) -> Result<(), crate::PyError> { - crate::module::thread::apply_all_thread_hooks(self)?; + if !crate::module::thread::all_thread_hooks_current(self) { + crate::module::thread::apply_all_thread_hooks(self)?; + } if majit_ir::eval_breaker_word::load() & majit_ir::eval_breaker_word::EB_ASYNC != 0 { let w_async_exception_type = crate::module::thread::take_async_exception(self as *mut ExecutionContext); @@ -720,6 +723,7 @@ impl ExecutionContext { /// per-frame trace is unset (so a later `frame.f_trace = cb` /// observes the unmodified instr_prev_plus_one and can fire its /// first `line` event correctly). + #[inline(always)] pub fn bytecode_only_trace(&mut self, frame: *mut PyFrame) -> Result<(), crate::PyError> { // PyPy has no object-space-null guard here: `space` is the interpreter // owner, not an optional tracing flag. Pyre represents that owner with diff --git a/pyre/pyre-interpreter/src/gateway.rs b/pyre/pyre-interpreter/src/gateway.rs index 7f390c68bfa..b3ac4a2ae26 100644 --- a/pyre/pyre-interpreter/src/gateway.rs +++ b/pyre/pyre-interpreter/src/gateway.rs @@ -498,6 +498,65 @@ pub const BUILTIN_CODE_GC_TYPE_ID: u32 = 13; /// pyre equivalent: returns Result so errors propagate through the call stack. pub type BuiltinCodeFn = fn(&[PyObjectRef]) -> Result; +/// Cold interp2app arity-error formatter. +/// +/// RPython's gateway constructs this exception only on the rejected-call +/// branch. Keep formatting out of generated wrapper JitCodes; valid calls +/// retain the argument-count guards but never execute this residual helper. +#[majit_macros::dont_look_inside] +pub fn method_arity_failure( + name: &str, + expected: &str, + given: usize, +) -> Result { + Err(crate::PyError::type_error(format!( + "{name}() takes {expected} ({given} given)" + ))) +} + +/// Cold zero-user-argument gateway failure. +/// +/// Valid generated wrappers guard their exact total arity before entering +/// this helper. The rejected branch still distinguishes CALL_KW's trailing +/// marker from surplus positional arguments, preserving the public gateway +/// error while keeping keyword classification out of the hot JitCode. +#[majit_macros::dont_look_inside] +pub fn method_noarg_failure( + args: &[PyObjectRef], + name: &str, + receiver_slots: usize, +) -> Result { + if crate::builtins::has_builtin_kwargs(args) { + Err(crate::PyError::type_error(format!( + "{name}() takes no keyword arguments" + ))) + } else { + method_arity_failure( + name, + "no arguments", + args.len().saturating_sub(receiver_slots), + ) + } +} + +/// Translation-visible registry of generated interp2app gateway bodies. +/// +/// RPython's `BuiltinCode.func` is a PBC whose possible function values are +/// discovered by the annotator and become the candidate graph list on the +/// eventual `indirect_call`. Rust erases that family to the bare +/// [`BuiltinCodeFn`] type, so `#[pyre_methods]` publishes the same set here. +/// The registry is process-global, matching the immutable generated function +/// objects; it is not runtime interpreter state. +#[derive(Clone, Copy)] +pub struct BuiltinWrapperDescriptor { + pub path: &'static str, + pub func: BuiltinCodeFn, +} + +#[cfg(not(target_arch = "wasm32"))] +#[linkme::distributed_slice] +pub static BUILTIN_WRAPPER_DESCRIPTORS: [BuiltinWrapperDescriptor]; + /// The type a method descriptor belongs to, and the layout test its receiver /// must satisfy — `PyDescrObject.d_type`, and the `self` entry of PyPy's /// `interp2app` unwrap_spec. diff --git a/pyre/pyre-interpreter/src/jit_fnaddr.rs b/pyre/pyre-interpreter/src/jit_fnaddr.rs index 66a422ed286..ef4a15d6553 100644 --- a/pyre/pyre-interpreter/src/jit_fnaddr.rs +++ b/pyre/pyre-interpreter/src/jit_fnaddr.rs @@ -333,6 +333,34 @@ pub fn is_list_write_barrier(addr: usize) -> bool { pub fn jit_trace_fnaddrs() -> Vec<(&'static str, i64)> { let mut entries = Vec::new(); + push_alias_pair( + &mut entries, + "pyre_interpreter::gateway::method_arity_failure", + "gateway::method_arity_failure", + crate::gateway::method_arity_failure as *const (), + ); + push_alias_pair( + &mut entries, + "pyre_interpreter::gateway::method_noarg_failure", + "gateway::method_noarg_failure", + crate::gateway::method_noarg_failure as *const (), + ); + push_alias_pair( + &mut entries, + "pyre_interpreter::builtins::builtin_kwargs_marker_dict", + "builtins::builtin_kwargs_marker_dict", + crate::builtins::builtin_kwargs_marker_dict as *const (), + ); + + // RPython annotator PBC parity for `BuiltinCode.func`: every generated + // interp2app wrapper is a possible value of the indirect function-pointer + // field. `#[pyre_methods]` contributes these process-global descriptors + // through the same link-time census used for pyre class descriptors. + #[cfg(not(target_arch = "wasm32"))] + for wrapper in crate::gateway::BUILTIN_WRAPPER_DESCRIPTORS { + push_fnaddr(&mut entries, wrapper.path, wrapper.func as *const ()); + } + push_alias_pair( &mut entries, "pyre_interpreter::runtime_ops::jit_make_function_from_globals", @@ -357,6 +385,22 @@ pub fn jit_trace_fnaddrs() -> Vec<(&'static str, i64)> { "pyre_interpreter::jit_sequence_getitem", crate::runtime_ops::jit_sequence_getitem as *const (), ); + // `rpython/rlib/rrandom.py Random.genrand32` contains the Mersenne + // Twister refill loops. `JitPolicy.look_inside_graph` deliberately + // rejects the loopy graph (it is not `@jit.unroll_safe`), so + // `Random.random` keeps two ordinary residual calls to the translated + // native helper. Publish that helper's address just as RPython's source + // translation/link step does; otherwise the codewriter can only emit a + // `symbolic_fnaddr_for_path` hash and an inline sub-walk must abort before + // reaching the native residual. + let random_genrand32: fn(&mut crate::module::_random::Random) -> u32 = + crate::module::_random::Random::genrand32; + push_alias_pair( + &mut entries, + "pyre_interpreter::module::_random::Random::genrand32", + "module::_random::Random::genrand32", + random_genrand32 as *const (), + ); push_alias_pair( &mut entries, "pyre_interpreter::runtime_ops::jit_next", @@ -3248,6 +3292,20 @@ mod tests { assert_eq!(bindings["pyre_object::jit_list_append"], list_append); } + #[test] + fn jit_trace_fnaddrs_covers_random_genrand32_residual() { + let bindings: HashMap<&'static str, i64> = jit_trace_fnaddrs().into_iter().collect(); + let genrand32: fn(&mut crate::module::_random::Random) -> u32 = + crate::module::_random::Random::genrand32; + let expected = genrand32 as *const () as usize as i64; + + assert_eq!( + bindings["pyre_interpreter::module::_random::Random::genrand32"], + expected + ); + assert_eq!(bindings["module::_random::Random::genrand32"], expected); + } + #[test] fn jit_trace_fnaddrs_covers_generated_runtime_helper_families() { let bindings: HashMap<&'static str, i64> = jit_trace_fnaddrs().into_iter().collect(); diff --git a/pyre/pyre-interpreter/src/module/_random/mod.rs b/pyre/pyre-interpreter/src/module/_random/mod.rs index e860b2c46b1..6c606b7e477 100644 --- a/pyre/pyre-interpreter/src/module/_random/mod.rs +++ b/pyre/pyre-interpreter/src/module/_random/mod.rs @@ -20,7 +20,7 @@ const MAGIC_CONSTANT_B: u32 = 19650218; const MAGIC_CONSTANT_C: u32 = 1664525; const MAGIC_CONSTANT_D: u32 = 1566083941; -struct Random { +pub(crate) struct Random { state: [u32; N], index: usize, } @@ -86,7 +86,7 @@ impl Random { if y & 1 != 0 { val ^ MATRIX_A } else { val } } - fn genrand32(&mut self) -> u32 { + pub(crate) fn genrand32(&mut self) -> u32 { if self.index >= N { let mt = &mut self.state; for kk in 0..(N - M) { diff --git a/pyre/pyre-interpreter/src/module/math/interp_math.rs b/pyre/pyre-interpreter/src/module/math/interp_math.rs index 7ac7b05953c..9fdbf8ba7b4 100644 --- a/pyre/pyre-interpreter/src/module/math/interp_math.rs +++ b/pyre/pyre-interpreter/src/module/math/interp_math.rs @@ -219,25 +219,136 @@ pm1_edom!(atanh, "expected a number between -1 and 1"); // Exponential / logarithmic pm1_edom!(sqrt, "expected a nonnegative input"); -/// True iff `callable` is the canonical builtin `math.sqrt` function object. -/// The JIT walker uses the builtin-code native fn-pointer identity to -/// distinguish it from a value rebound under the same `math.sqrt` name, so a -/// monkeypatched `math.sqrt` correctly declines the pure-inline specialization. -pub fn is_math_sqrt_function(callable: PyObjectRef) -> bool { +static MATH_SQRT_WRAPPER: std::sync::OnceLock = std::sync::OnceLock::new(); +static MATH_FREXP_WRAPPER: std::sync::OnceLock = std::sync::OnceLock::new(); +static MATH_LDEXP_WRAPPER: std::sync::OnceLock = std::sync::OnceLock::new(); +static MATH_ISQRT_WRAPPER: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Record the checked-arity wrapper pointers installed by `py_module!`. +/// +/// `py_checked_arity_fn!` wraps each `interp_math` body in a non-capturing +/// closure, so a BuiltinCode stores the wrapper pointer rather than (for +/// example) [`frexp`] itself. The wrappers are immutable process code, hence +/// process-global `OnceLock` is the same ownership shape as pyre's +/// other immortal runtime metadata and needs no GC rooting. +pub fn register_jit_builtin_wrappers(ns: PyObjectRef) { + for (name, slot) in [ + ("sqrt", &MATH_SQRT_WRAPPER), + ("frexp", &MATH_FREXP_WRAPPER), + ("ldexp", &MATH_LDEXP_WRAPPER), + ("isqrt", &MATH_ISQRT_WRAPPER), + ] { + let callable = crate::module_ns_get(ns, name) + .unwrap_or_else(|| panic!("math.{name} missing after module registration")); + let wrapper = unsafe { + let code = crate::function_get_code(callable) as PyObjectRef; + debug_assert!(crate::gateway::is_builtin_code(code)); + crate::gateway::builtin_code_get(code) as usize + }; + let installed = slot.get_or_init(|| wrapper); + debug_assert_eq!(*installed, wrapper); + } +} + +unsafe fn math_builtin_wrapper_matches( + callable: PyObjectRef, + expected: &std::sync::OnceLock, +) -> bool { unsafe { if callable.is_null() || !crate::is_function(callable) { return false; } let code = crate::function_get_code(callable) as PyObjectRef; - if code.is_null() || !crate::gateway::is_builtin_code(code) { - return false; - } - std::ptr::fn_addr_eq( - crate::gateway::builtin_code_get(code), - sqrt as crate::gateway::BuiltinCodeFn, - ) + !code.is_null() + && crate::gateway::is_builtin_code(code) + && expected + .get() + .is_some_and(|addr| *addr == crate::gateway::builtin_code_get(code) as usize) } } + +/// True iff `callable` is the canonical builtin `math.sqrt` function object. +/// The JIT walker uses the builtin-code native fn-pointer identity to +/// distinguish it from a value rebound under the same `math.sqrt` name, so a +/// monkeypatched `math.sqrt` correctly declines the pure-inline specialization. +pub fn is_math_sqrt_function(callable: PyObjectRef) -> bool { + unsafe { math_builtin_wrapper_matches(callable, &MATH_SQRT_WRAPPER) } +} + +/// Callable-identity probes used by the meta-trace walker. As with +/// [`is_math_sqrt_function`], compare the immutable BuiltinCode function +/// pointer rather than a module/name string so rebinding `math.frexp` or +/// `math.ldexp` cannot enter a specialization for the old callable. +pub fn is_math_frexp_function(callable: PyObjectRef) -> bool { + unsafe { math_builtin_wrapper_matches(callable, &MATH_FREXP_WRAPPER) } +} + +pub fn is_math_ldexp_function(callable: PyObjectRef) -> bool { + unsafe { math_builtin_wrapper_matches(callable, &MATH_LDEXP_WRAPPER) } +} + +pub fn is_math_isqrt_function(callable: PyObjectRef) -> bool { + unsafe { math_builtin_wrapper_matches(callable, &MATH_ISQRT_WRAPPER) } +} + +/// Raw, allocation-free counterparts of RPython's `ll_math_frexp` result +/// components. The translated PyPy trace carries the pair as two unboxed +/// values before `space.newtuple2` virtualizes; pyre's walker emits one pure +/// call per component because the IR has no multi-result call opcode. +pub extern "C" fn jit_math_frexp_mantissa(x: f64) -> f64 { + pymath::math::frexp(x).0 +} + +pub extern "C" fn jit_math_frexp_exponent(x: f64) -> i64 { + pymath::math::frexp(x).1 as i64 +} + +/// Non-raising raw `ldexp` for a guarded JIT fast path. RPython lowers +/// `ll_math_ldexp` to the platform operation and guards its exceptional +/// result. On overflow, return signed infinity so the walker's finite-result +/// guard deoptimizes and the ordinary builtin re-executes to raise +/// `OverflowError`; successful finite/underflow results are returned exactly. +pub extern "C" fn jit_math_ldexp_raw(x: f64, exp: i64) -> f64 { + if x == 0.0 || !x.is_finite() { + return x; + } + let Ok(exp) = i32::try_from(exp) else { + return if exp < 0 { + 0.0f64.copysign(x) + } else { + f64::INFINITY.copysign(x) + }; + }; + match pymath::math::ldexp(x, exp) { + Ok(result) => result, + Err(_) => f64::INFINITY.copysign(x), + } +} + +/// Allocation-free exact machine-integer arm of `app_math.isqrt`. +/// +/// PyPy meta-traces the app-level implementation with an unboxed Signed when +/// the argument is a `W_IntObject`. The native wrapper otherwise promotes +/// that value to `rbigint` before running the same algorithm, hiding the +/// unboxed arm from pyre's walker. Use a hardware square-root estimate and +/// exact division-based corrections; the latter preserve integer semantics +/// even where `f64` cannot represent every input. +pub extern "C" fn jit_math_isqrt_i64(n: i64) -> i64 { + debug_assert!(n >= 0); + if n == 0 { + return 0; + } + let n = n as u64; + let mut root = (n as f64).sqrt() as u64; + while root + 1 <= n / (root + 1) { + root += 1; + } + while root > n / root { + root -= 1; + } + root as i64 +} + pm1!(cbrt); pm1!(exp); pm1!(exp2); @@ -874,12 +985,105 @@ pub fn isqrt(args: &[PyObjectRef]) -> PyResult { } pub fn fsum(args: &[PyObjectRef]) -> PyResult { - let items = crate::builtins::collect_iterable(args[0])?; - let floats: Vec = items - .iter() - .map(|&a| try_get_double(a)) - .collect::, _>>()?; - map_err(pymath::math::fsum(floats)) + // interp_math.py:572-633: consume the iterator once while maintaining the + // partials array. The old port first materialized every boxed element and + // then built a second Vec; besides diverging from upstream, that kept + // one shadow-stack root per input alive until the entire iterable had been + // consumed. + let _roots = pyre_object::gc_roots::push_roots(); + let iterable_slot = pyre_object::gc_roots::shadow_stack_len(); + pyre_object::gc_roots::pin_root(args[0]); + let w_iter = crate::baseobjspace::iter(pyre_object::gc_roots::shadow_stack_get(iterable_slot))?; + let iter_slot = pyre_object::gc_roots::shadow_stack_len(); + pyre_object::gc_roots::pin_root(w_iter); + + let mut inf_sum = 0.0; + let mut special_sum = 0.0; + let mut partials: Vec = Vec::new(); + loop { + let w_value = + match crate::baseobjspace::next(pyre_object::gc_roots::shadow_stack_get(iter_slot)) { + Ok(value) => value, + Err(err) if err.kind == crate::PyErrorKind::StopIteration => break, + Err(err) => return Err(err), + }; + // `_get_double` can invoke user code. Keep the yielded object rooted + // only across that conversion, exactly like the translated livevar at + // this point in the upstream loop. + let original = { + let _value_root = pyre_object::gc_roots::push_roots(); + let value_slot = pyre_object::gc_roots::shadow_stack_len(); + pyre_object::gc_roots::pin_root(w_value); + try_get_double(pyre_object::gc_roots::shadow_stack_get(value_slot))? + }; + let mut value = original; + let mut added = 0; + for index in 0..partials.len() { + let mut partial = partials[index]; + if value.abs() < partial.abs() { + std::mem::swap(&mut value, &mut partial); + } + let hi = value + partial; + let yr = hi - value; + let lo = partial - yr; + if lo != 0.0 { + partials[added] = lo; + added += 1; + } + value = hi; + } + partials.truncate(added); + if value != 0.0 { + if !value.is_finite() { + if original.is_finite() { + return map_err(Err(pymath::Error::ERANGE)); + } + if original.is_infinite() { + inf_sum += original; + } + special_sum += original; + partials.clear(); + } else { + partials.push(value); + } + } + } + + if special_sum != 0.0 { + if inf_sum.is_nan() { + return map_err(Err(pymath::Error::EDOM)); + } + return Ok(floatobject::w_float_new(special_sum)); + } + let mut hi = 0.0; + let mut lo = 0.0; + let mut index = partials.len(); + if index > 0 { + index -= 1; + hi = partials[index]; + while index > 0 { + index -= 1; + let value = hi; + let partial = partials[index]; + hi = value + partial; + let yr = hi - value; + lo = partial - yr; + if lo != 0.0 { + break; + } + } + if index > 0 + && ((lo < 0.0 && partials[index - 1] < 0.0) || (lo > 0.0 && partials[index - 1] > 0.0)) + { + let doubled = lo * 2.0; + let value = hi + doubled; + let yr = value - hi; + if doubled == yr { + hi = value; + } + } + } + Ok(floatobject::w_float_new(hi)) } pub fn prod(args: &[PyObjectRef]) -> PyResult { diff --git a/pyre/pyre-interpreter/src/module/math/mod.rs b/pyre/pyre-interpreter/src/module/math/mod.rs index f9cf330ab9d..a7df8cad4d4 100644 --- a/pyre/pyre-interpreter/src/module/math/mod.rs +++ b/pyre/pyre-interpreter/src/module/math/mod.rs @@ -94,4 +94,7 @@ crate::py_module! { "perm" / * = m::perm, "isqrt" / 1 = m::isqrt, }, + extra_init: |ns| { + m::register_jit_builtin_wrappers(ns); + }, } diff --git a/pyre/pyre-interpreter/src/module/thread/mod.rs b/pyre/pyre-interpreter/src/module/thread/mod.rs index f15a765315a..15ce666cf2a 100644 --- a/pyre/pyre-interpreter/src/module/thread/mod.rs +++ b/pyre/pyre-interpreter/src/module/thread/mod.rs @@ -58,6 +58,7 @@ pub fn set_finalizing() { majit_gc::gc_sync::request_stw(|_| { FINALIZING_THREAD.store(ident, Ordering::Release); FINALIZING.store(true, Ordering::Release); + majit_ir::eval_breaker_word::set_finalizing(); }); } @@ -203,6 +204,19 @@ pub(crate) fn set_profile_all_execution_contexts( Ok(()) } +/// Fast bytecode-boundary gate for CPython's all-thread tracing extensions. +/// +/// PyPy's ordinary `bytecode_trace` common path contains only its trace check +/// and action ticker. Pyre additionally has to notice process-wide +/// `_settraceallthreads` / `_setprofileallthreads` generations, but when +/// neither changed it need not enter the updater (and its mutex-bearing slow +/// arms) at every opcode. +#[inline(always)] +pub(crate) fn all_thread_hooks_current(ec: &crate::PyExecutionContext) -> bool { + ec.trace_all_generation == TRACE_ALL_GENERATION.load(Ordering::Acquire) + && ec.profile_all_generation == PROFILE_ALL_GENERATION.load(Ordering::Acquire) +} + pub(crate) fn apply_all_thread_hooks( ec: &mut crate::PyExecutionContext, ) -> Result<(), crate::PyError> { diff --git a/pyre/pyre-jit-trace/build.rs b/pyre/pyre-jit-trace/build.rs index 6683e0d12b1..98cbf9f48aa 100644 --- a/pyre/pyre-jit-trace/build.rs +++ b/pyre/pyre-jit-trace/build.rs @@ -11,7 +11,7 @@ use walkdir::WalkDir; #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; -const CODEGEN_CACHE_VERSION: &str = "pyre-jit-trace-codegen-cache-v3"; +const CODEGEN_CACHE_VERSION: &str = "pyre-jit-trace-codegen-cache-v4"; /// Retained cache entries. Each is ~6 MB, and a handful covers the /// configurations one checkout switches between (native/wasm × release/dev). const CODEGEN_CACHE_MAX_ENTRIES: usize = 8; @@ -21,6 +21,7 @@ const CODEGEN_OUTPUTS: &[&str] = &[ "jit_trace_gen.rs", "jit_metadata.json", "jitcodes.bin", + "indirectcalltargets.bin", "jit_drivers.bin", "insns.bin", "descrs.bin", @@ -87,6 +88,11 @@ fn emit_llbc_extraction_placeholders() { .unwrap(), ) .unwrap(); + std::fs::write( + format!("{out_dir}/indirectcalltargets.bin"), + bincode::serialize(&Vec::::new()).unwrap(), + ) + .unwrap(); std::fs::write( format!("{out_dir}/jit_drivers.bin"), bincode::serialize(&Vec::::new()).unwrap(), @@ -500,6 +506,12 @@ fn real_main() { // codewriter.make_jitcodes()`. let jitcodes_bin = bincode::serialize(&pipeline.jitcodes).unwrap(); std::fs::write(format!("{out_dir}/jitcodes.bin"), &jitcodes_bin).unwrap(); + let indirectcalltargets_bin = bincode::serialize(&pipeline.indirectcalltarget_indices).unwrap(); + std::fs::write( + format!("{out_dir}/indirectcalltargets.bin"), + &indirectcalltargets_bin, + ) + .unwrap(); // Persist the explicit portal → main-JitCode mapping. Runtime consumes // this directly instead of rediscovering the portal through name or flag diff --git a/pyre/pyre-jit-trace/src/descr.rs b/pyre/pyre-jit-trace/src/descr.rs index 4057d9f288a..60cbfa83293 100644 --- a/pyre/pyre-jit-trace/src/descr.rs +++ b/pyre/pyre-jit-trace/src/descr.rs @@ -913,6 +913,61 @@ static RANGE_DESCR_GROUP: LazyLock = LazyLock::new(|| { ) }); +/// `Function.defs_w` — PyPy `function.py:47` +/// `_immutable_fields_ = [..., 'defs_w?[*]', ...]`. +/// +/// The `?` makes the field quasi-immutable upstream and `[*]` makes the +/// selected defaults immutable after the field has been promoted. Pyre's +/// `function_set_defaults` does not yet call `do_force_quasi_immutable`, so +/// marking this descriptor quasi-immutable would leave compiled loops alive +/// after `f.__defaults__ = ...`. Keep the field live/mutable for now; the +/// inline-call path pairs its read with a `GuardValue`, which is the sound +/// pre-invalidation equivalent. The tuple's backing array has its own +/// immutable descriptor and is read with `GetarrayitemGcPureR`. +static FUNCTION_DESCR_GROUP: LazyLock = LazyLock::new(|| { + build_object_descr_group_with_def_path( + pyre_interpreter::function::FUNCTION_OBJECT_SIZE, + FUNCTION_GC_TYPE_ID, + &pyre_interpreter::FUNCTION_TYPE as *const _ as usize, + &[( + "defs_w", + pyre_interpreter::function::FUNCTION_DEFS_W_OFFSET, + std::mem::size_of::(), + Type::Ref, + false, + false, + false, + )], + "Function", + "function::Function", + ) +}); + +/// `W_DictObject.keys_version` is pyre's explicit representation of the live +/// strategy-iterator state PyPy carries implicitly +/// (`dictmultiobject.py:807-845`). Key insertion/removal/strategy replacement +/// bumps it; value-only replacement deliberately does not. A promoted +/// identity-key lookup can therefore guard this field to pin the resolved +/// entry index while continuing to read that entry's value live. +static W_DICT_DESCR_GROUP: LazyLock = LazyLock::new(|| { + build_object_descr_group_with_def_path( + pyre_object::dictmultiobject::W_DICT_OBJECT_SIZE, + W_DICT_GC_TYPE_ID, + &pyre_object::pyobject::DICT_TYPE as *const _ as usize, + &[( + "keys_version", + std::mem::offset_of!(pyre_object::dictmultiobject::W_DictObject, keys_version), + std::mem::size_of::(), + Type::Int, + false, + false, + false, + )], + "W_DictObject", + "dictmultiobject::W_DictObject", + ) +}); + /// `Method` field layout — `w_function`, `w_self`, `w_class` per /// `function.rs:9-15`. All three are Ref slots; the JIT only consumes /// `w_function` (for guarding which method) and `w_self` (for recovering @@ -1850,6 +1905,17 @@ pub fn method_w_function_descr() -> DescrRef { field_descr_from_group(&W_METHOD_DESCR_GROUP, 0) } +/// Live `Function.defs_w` field used by the positional-default inline path. +/// See [`FUNCTION_DESCR_GROUP`] for why this is deliberately mutable until +/// pyre wires the upstream quasi-immutable invalidation hook. +pub fn function_defs_w_descr() -> DescrRef { + field_descr_from_group(&FUNCTION_DESCR_GROUP, 0) +} + +pub fn dict_keys_version_descr() -> DescrRef { + field_descr_from_group(&W_DICT_DESCR_GROUP, 0) +} + /// `Method.w_self` — the receiver object. The bound-method /// specialization extracts this via `GetfieldGcR` to recover the receiver /// `OpRef` after `LOAD_METHOD` discarded it (load_method.rs:6334 pushes diff --git a/pyre/pyre-jit-trace/src/helpers.rs b/pyre/pyre-jit-trace/src/helpers.rs index 49589558e99..21f897ea91f 100644 --- a/pyre/pyre-jit-trace/src/helpers.rs +++ b/pyre/pyre-jit-trace/src/helpers.rs @@ -106,6 +106,21 @@ pub fn emit_trace_call_ref_typed( ctx.call_ref_typed_with_effect(helper, args, arg_types, default_effect_info()) } +/// Read the value at a promoted exact-dict entry index. The caller guards +/// `W_DictObject.keys_version`, the explicit pyre representation of PyPy's +/// live strategy-iterator state, so the index remains attached to the same +/// identity key. Value overwrites do not bump that version and are observed +/// by this live read, matching the r_dict entry-value load. +pub extern "C" fn jit_dict_nth_value(dict: i64, index: i64) -> i64 { + unsafe { + pyre_object::dictmultiobject::w_dict_nth_item( + dict as pyre_object::PyObjectRef, + index as usize, + ) + .map_or(0, |(_, value)| value as i64) + } +} + pub fn emit_trace_call_ref_typed_elidable_cannot_raise( ctx: &mut TraceCtx, helper: *const (), @@ -1041,7 +1056,10 @@ pub fn emit_box_float_inline( /// already-boxed positional argument refs. Same field-complete frame shape as /// [`emit_new_pyframe_inline_self_recursive`] but seeds `locals[0..nparams]` /// from `param_boxes` (Ref boxes at the Python call boundary) instead of -/// boxing a single raw int. The frame is the callee MIFrame's `frame` red — +/// boxing a single raw int. Existing closure cells are placed at +/// `freevar_start..`, matching `PyFrame::finish_for_call_with_globals_obj`; +/// callers that need fresh cellvars remain on the residual path. The frame is +/// the callee MIFrame's `frame` red — /// `_opimpl_inline_call*` / `perform_call`+`setup_call` create a fresh frame /// per inlined call (`pyjitpl.py:2445-2476,1862-1874`); the box stays virtual /// on the hot path (the optimizer folds `NewWithVtable`+`SetfieldGc`) and is @@ -1051,6 +1069,8 @@ pub fn emit_box_float_inline( pub fn emit_new_pyframe_inline_with_params( ctx: &mut TraceCtx, param_boxes: &[OpRef], + freevar_cells: &[OpRef], + freevar_start: usize, array_size: usize, valuestackdepth: usize, pycode: OpRef, @@ -1103,6 +1123,18 @@ pub fn emit_new_pyframe_inline_with_params( ); ctx.heapcache_setarrayitem(locals_array, idx, heapcache_item_descr_index, p); } + // PyFrame.finish_for_call_with_globals_obj: a closure contributes the + // existing cell objects themselves after locals + pure cellvars. LOAD_DEREF + // must therefore read the live cell, not a snapshot of its contents. + for (i, &cell) in freevar_cells.iter().enumerate() { + let idx = ctx.const_int((freevar_start + i) as i64); + ctx.record_op_with_descr( + OpCode::SetarrayitemGc, + &[locals_array, idx, cell], + array_descr.clone(), + ); + ctx.heapcache_setarrayitem(locals_array, idx, heapcache_item_descr_index, cell); + } let new_frame = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], pyframe_size_descr()); ctx.heap_cache_mut().new_object(new_frame); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index f5ace365db5..9c1ed93fb68 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -1232,7 +1232,11 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( // aborts before the hazardous body is committed. Every other nested // residual inlines. The hazard scan is last so the cheap checks // short-circuit it. - let nested = !in_selfrec_fold + // A carrier-resume sub-walk starts at the failed guard; it does not replay + // an enclosing CALL. RPython resumes residual calls at every rebuilt + // framestack depth, so this forward-capture hazard excludes the carrier. + let nested = !ctx.fbw_mode.carrier_resume + && !in_selfrec_fold && !in_exception_string_inline && !ctx.session.borrow().framestack.is_empty(); let hazardous_callee = if nested && foriter_deferred_inline.is_none() { @@ -1734,15 +1738,22 @@ pub(crate) enum CalleeReplaySafety { /// register reaching a join can hold whichever allocation the taken path put /// there, which this straight-line scan cannot name. /// +/// Exact numeric provenance for one positional parameter of an inline callee. +/// The two facts stay separate because bitwise specialization accepts only +/// exact ints, while add/subtract/multiply also accept exact floats. +#[derive(Clone, Copy, Default)] +pub(crate) struct ExactNumericArg { + pub(crate) numeric: bool, + pub(crate) plain_int: bool, +} + /// The `BINARY_OP` exemption needs the mirror-image fact — which values came -/// FROM the caller, the only ones `args_all_exact_numeric` / -/// `args_all_exact_plain_int` describe — so parameter provenance is tracked -/// alongside freshness; see `arg_backed_ref_regs` below. +/// FROM exact-numeric caller arguments — so parameter provenance is tracked +/// slot-by-slot alongside freshness. This matters for method-form calls: +/// `self` is commonly nonnumeric while a later argument is an exact int. pub(crate) fn fbw_callee_body_replay_safety( body_code: &[u8], - nparams: usize, - args_all_exact_numeric: bool, - args_all_exact_plain_int: bool, + exact_numeric_args: &[ExactNumericArg], num_regs_i: usize, constants_i: &[i64], num_regs_r: usize, @@ -1759,8 +1770,8 @@ pub(crate) fn fbw_callee_body_replay_safety( // can neither dispatch to a user `__add__` nor be mutated in place, so the // op commits nothing a replay could double. Three sources close over it: // - // - an incoming argument, which the caller checked is an exact int or float - // (`args_all_exact_*`). A body does not receive parameters in registers; + // - an incoming argument, which the caller checked is an exact int or float. + // A body does not receive parameters in registers; // it reads them out of its own frame, whose `localsplus` slots // `[0, nparams)` the exact-positional entry convention binds from the // passed args (`callee_args.len() == nparams`, closure-free). So the @@ -1784,27 +1795,34 @@ pub(crate) fn fbw_callee_body_replay_safety( // freshness does: a slot or register reaching a join holds whatever the // taken path put there, which this straight-line scan cannot name. The // constant pool is re-seeded across that reset, being immutable. - let mut seed_ref_regs = [false; u8::MAX as usize + 1]; + let mut seed_numeric_ref_regs = [false; u8::MAX as usize + 1]; + let mut seed_plain_int_ref_regs = [false; u8::MAX as usize + 1]; for (index, &raw) in constants_r.iter().enumerate() { let Some(reg) = num_regs_r .checked_add(index) - .filter(|r| *r < seed_ref_regs.len()) + .filter(|r| *r < seed_numeric_ref_regs.len()) else { break; }; let obj = raw as usize as pyre_object::PyObjectRef; - seed_ref_regs[reg] = !obj.is_null() - && unsafe { - pyre_object::is_plain_int1(obj) || pyre_object::is_plain_float_strict(obj) - }; + if !obj.is_null() { + let exact_int = unsafe { pyre_object::is_plain_int1(obj) }; + seed_plain_int_ref_regs[reg] = exact_int; + seed_numeric_ref_regs[reg] = + exact_int || unsafe { pyre_object::is_plain_float_strict(obj) }; + } } - let mut binop_safe_ref_regs = seed_ref_regs; - let mut binop_safe_slots = [false; BODY_TRACKED_FRAME_SLOTS]; - for slot in binop_safe_slots - .iter_mut() - .take(nparams.min(BODY_TRACKED_FRAME_SLOTS)) + let mut numeric_ref_regs = seed_numeric_ref_regs; + let mut plain_int_ref_regs = seed_plain_int_ref_regs; + let mut numeric_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + let mut plain_int_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + for (slot, exact) in exact_numeric_args + .iter() + .take(BODY_TRACKED_FRAME_SLOTS) + .enumerate() { - *slot = true; + numeric_slots[slot] = exact.numeric; + plain_int_slots[slot] = exact.plain_int; } // The frame register every vable op in this body has used so far. A second // one would mean the slot bookkeeping above is tracking two different @@ -1815,15 +1833,18 @@ pub(crate) fn fbw_callee_body_replay_safety( while pc < body_code.len() { if branch_targets.contains(&pc) { fresh_ref_regs = [false; u8::MAX as usize + 1]; - binop_safe_ref_regs = seed_ref_regs; - binop_safe_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + numeric_ref_regs = seed_numeric_ref_regs; + plain_int_ref_regs = seed_plain_int_ref_regs; + numeric_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + plain_int_slots = [false; BODY_TRACKED_FRAME_SLOTS]; } let Some(d) = crate::jitcode_runtime::decode_op_at(body_code, pc) else { return CalleeReplaySafety::Dirty; }; // Set by the arms below when this op's `>r` result is itself an // immutable builtin. - let mut dst_binop_safe = false; + let mut dst_exact_numeric = false; + let mut dst_exact_plain_int = false; // The ref-slot accessors name the frame in operand 0 and the slot in // operand 1, both one byte wide. The `_i` / `_f` variants address a @@ -1846,14 +1867,21 @@ pub(crate) fn fbw_callee_body_replay_safety( // resolve could name any of them, so it drops the lot. match vable_slot { Some(slot) if slot < BODY_TRACKED_FRAME_SLOTS => { - binop_safe_slots[slot] = d.argcodes.starts_with("rir") - && body_code - .get(d.pc + 3) - .is_some_and(|src| binop_safe_ref_regs[*src as usize]); + let src = d + .argcodes + .starts_with("rir") + .then(|| body_code.get(d.pc + 3)) + .flatten() + .copied(); + numeric_slots[slot] = src.is_some_and(|src| numeric_ref_regs[src as usize]); + plain_int_slots[slot] = src.is_some_and(|src| plain_int_ref_regs[src as usize]); } // Past the tracked window: cannot alias a slot inside it. Some(_) => {} - None => binop_safe_slots = [false; BODY_TRACKED_FRAME_SLOTS], + None => { + numeric_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + plain_int_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + } } } @@ -1888,35 +1916,35 @@ pub(crate) fn fbw_callee_body_replay_safety( | majit_ir::PyreHelperKind::NewtupleFromArray | majit_ir::PyreHelperKind::NewlistFromArray ); - // A code constant is compiler-produced, and `box_int` builds a - // brand-new `W_IntObject`; neither can be an instance of a user - // class, so both are usable as a proven binop operand. The two - // array consumers build a tuple and a LIST — the list is mutable, - // so only the tuple qualifies — and `load_global` reads whatever - // the module namespace holds, which is the very hole being closed. - let dst_immutable_builtin = matches!( - ei.pyre_helper, - majit_ir::PyreHelperKind::LoadConst - | majit_ir::PyreHelperKind::BoxInt - | majit_ir::PyreHelperKind::NewtupleFromArray - ); + // `box_int` is the only generic replay-safe helper here whose + // result is necessarily numeric. `load_const` may return a str, + // tuple, or another nonnumeric immutable value, while the typed + // jitcode constant pool was classified exactly above. + let dst_boxed_int = ei.pyre_helper == majit_ir::PyreHelperKind::BoxInt; let provably_side_effect_free = replay_safe_read || ei.check_is_elidable() || ei.extraeffect == majit_ir::ExtraEffect::LoopInvariant; let accepted_binop = !provably_side_effect_free && residual_call_is_specialized_plain_numeric_binop( body_code, - args_all_exact_numeric, - args_all_exact_plain_int, - &binop_safe_ref_regs, + &numeric_ref_regs, + &plain_int_ref_regs, &d, num_regs_i, constants_i, callee_descr_refs, ); - // A builtin binary op over immutable builtin operands returns one, - // so its result may serve as an operand of the next. - dst_binop_safe = dst_immutable_builtin || accepted_binop; + // An accepted arithmetic op over exact numeric operands returns an + // exact numeric. Bitwise ops require and return exact ints. + dst_exact_numeric = dst_boxed_int || accepted_binop; + dst_exact_plain_int = dst_boxed_int + || (accepted_binop + && residual_call_is_specialized_plain_int_binop( + body_code, + &d, + num_regs_i, + constants_i, + )); if !provably_side_effect_free && !accepted_binop { // A Python-level CALL is the one shape this scan cannot // settle: the inline lever binds its callee only at the call, @@ -1933,7 +1961,8 @@ pub(crate) fn fbw_callee_body_replay_safety( deferred_call = true; // The callee this resolves to is a runtime value, so what it // can reach through this frame is unknown here. - binop_safe_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + numeric_slots = [false; BODY_TRACKED_FRAME_SLOTS]; + plain_int_slots = [false; BODY_TRACKED_FRAME_SLOTS]; } else { return CalleeReplaySafety::Dirty; } @@ -1999,15 +2028,24 @@ pub(crate) fn fbw_callee_body_replay_safety( // frame slot, from the residual arm above, or through a verbatim // copy. Every other producer overwrites the register with an // unproven value. - binop_safe_ref_regs[dst as usize] = dst_binop_safe + numeric_ref_regs[dst as usize] = dst_exact_numeric + || vable_slot.is_some_and(|slot| { + d.opname.starts_with("getarrayitem_vable_r") + && numeric_slots.get(slot).copied().unwrap_or(false) + }) + || (d.key == "ref_copy/r>r" + && body_code + .get(d.pc + 1) + .is_some_and(|src| numeric_ref_regs[*src as usize])); + plain_int_ref_regs[dst as usize] = dst_exact_plain_int || vable_slot.is_some_and(|slot| { d.opname.starts_with("getarrayitem_vable_r") - && binop_safe_slots.get(slot).copied().unwrap_or(false) + && plain_int_slots.get(slot).copied().unwrap_or(false) }) || (d.key == "ref_copy/r>r" && body_code .get(d.pc + 1) - .is_some_and(|src| binop_safe_ref_regs[*src as usize])); + .is_some_and(|src| plain_int_ref_regs[*src as usize])); } pc = d.next_pc; } diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index e9f3739cfc1..ceda49c6bb1 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -16,6 +16,57 @@ use super::*; +#[derive(Clone, Copy)] +struct BoundMethodInline { + method_op: OpRef, + function: pyre_object::PyObjectRef, + receiver: pyre_object::PyObjectRef, +} + +struct PositionalDefaultsInline { + tuple: pyre_object::PyObjectRef, + /// `(parameter index, tuple index, concrete value)`. + values: Vec<(usize, usize, pyre_object::PyObjectRef)>, +} + +/// `function.py:188-193,217-231` — determine the tail of `defs_w` used by a +/// flat positional call with missing arguments. The trace currently accepts +/// the ordinary translated `W_TupleObject` representation only: its +/// `wrappeditems[*]` storage has the exact immutable-array shape upstream's +/// `defs_w?[*]` lowers to. Specialised numeric tuples are a different +/// unboxed representation and safely remain on the residual call path. +unsafe fn positional_defaults_for_inline( + callable: pyre_object::PyObjectRef, + nargs: usize, + nparams: usize, +) -> Option { + if nargs >= nparams { + return None; + } + let tuple = unsafe { pyre_interpreter::function_get_defaults(callable) }; + if tuple.is_null() + || !std::ptr::eq( + unsafe { (*tuple).ob_type }, + &pyre_object::pyobject::TUPLE_TYPE, + ) + { + return None; + } + let ndefaults = unsafe { pyre_object::w_tuple_len(tuple) }; + let missing = nparams - nargs; + if missing > ndefaults { + return None; + } + let start = ndefaults - missing; + let mut values = Vec::with_capacity(missing); + for offset in 0..missing { + let tuple_index = start + offset; + let value = unsafe { pyre_object::w_tuple_getitem(tuple, tuple_index as i64) }?; + values.push((nargs + offset, tuple_index, value)); + } + Some(PositionalDefaultsInline { tuple, values }) +} + /// Path-1 (#68): resolve a scalar `getfield_vable_r` read off an inlined /// callee's OWN (unseeded) portal frame to the callee's compile-time /// constant. This is the walk-time mirror of the codewriter's non-portal @@ -841,6 +892,8 @@ pub(crate) fn try_walker_call_assembler_self_recursive( let callee_frame = crate::helpers::emit_new_pyframe_inline_with_params( ctx.trace_ctx, ¶m_boxes, + &[], + 0, nlocals + max_stack, nlocals, pycode_const, @@ -1249,7 +1302,9 @@ pub(crate) fn reconstructed_all_ref_call_stack( /// /// `r_args` layout is `[callable, self_or_null, kwnames, arg0..argN-1]`; the /// trailing `nkw` args are the keyword values, `kwnames[j]` naming -/// `arg[n_pos + j]` where `n_pos = nargs - nkw`. +/// `arg[n_pos + j]` where `n_pos = nargs - nkw`. `receiver`, when present, +/// is the implicit leading argument inserted by `call_kw` for method-form +/// calls (`Arguments.prepend` / `funcrun_obj` upstream). /// /// Returns `None` (declining to the residual call, no behavior change) for any /// shape the plain positional seeding cannot serve: a non-constant / non-tuple @@ -1285,6 +1340,7 @@ unsafe fn fbw_reorder_call_kw_args( arg_concretes: &[ConcreteValue], w_code: *const (), nparams: usize, + receiver: Option<(OpRef, ConcreteValue)>, ) -> Option<(Vec, Vec)> { if r_args.len() < 3 || arg_concretes.len() < 3 { return None; @@ -1305,7 +1361,8 @@ unsafe fn fbw_reorder_call_kw_args( // Every positional parameter must be filled exactly once from a passed arg: // no defaults, no *args/**kwargs/keyword-only slots the seeding would leave // unbound. - if nparams == 0 || nkw > nargs || nargs != nparams { + let receiver_count = usize::from(receiver.is_some()); + if nparams == 0 || nkw > nargs || nargs + receiver_count != nparams { return None; } let raw = unsafe { @@ -1325,9 +1382,14 @@ unsafe fn fbw_reorder_call_kw_args( let n_pos = nargs - nkw; let mut slot_args: Vec> = vec![None; nparams]; let mut slot_conc: Vec> = vec![None; nparams]; + if let Some((receiver_arg, receiver_concrete)) = receiver { + slot_args[0] = Some(receiver_arg); + slot_conc[0] = Some(receiver_concrete); + } for k in 0..n_pos { - slot_args[k] = Some(args[k]); - slot_conc[k] = Some(arg_conc[k]); + let pi = receiver_count + k; + slot_args[pi] = Some(args[k]); + slot_conc[pi] = Some(arg_conc[k]); } for j in 0..nkw { let name_obj = unsafe { pyre_object::w_tuple_getitem(kwnames, j as i64) }?; @@ -1345,7 +1407,9 @@ unsafe fn fbw_reorder_call_kw_args( // argument"). A name in the positional-only range is not bindable by // keyword at all — `def f(x, /)` called as `f(x=1)` is a TypeError, so // binding slot 0 here would inline a call the interpreter rejects. - if pi < n_pos || pi < unsafe { (*raw).posonlyarg_count } as usize || slot_args[pi].is_some() + if pi < receiver_count + n_pos + || pi < unsafe { (*raw).posonlyarg_count } as usize + || slot_args[pi].is_some() { return None; } @@ -1540,8 +1604,30 @@ pub(crate) fn try_walker_inline_user_call( ConcreteValue::Null => pyre_object::PY_NULL, _ => return Ok(None), }; - let method_form = !null_or_self.is_null() && null_or_self != pyre_object::PY_NULL; - let Some((w_code, nparams, has_closure)) = (unsafe { resolve_inlinable_callee(callable) }) + let mut method_form = !null_or_self.is_null() && null_or_self != pyre_object::PY_NULL; + // baseobjspace.py:1254-1259 unwraps `_Method` before the Function + // valuestack fast path. CALLs through a stored bound method (notably the + // module aliases in random.py) arrive as `[Method, PY_NULL, args...]`, so + // recover its immutable function/receiver fields and feed the same + // method-form callee shape used by LOAD_METHOD. + let bound_method = if !method_form && unsafe { pyre_object::is_method(callable) } { + let function = unsafe { pyre_object::w_method_get_func(callable) }; + let receiver = unsafe { pyre_object::w_method_get_self(callable) }; + if function.is_null() || receiver.is_null() { + return Ok(None); + } + method_form = true; + Some(BoundMethodInline { + method_op: r_args[0], + function, + receiver, + }) + } else { + None + }; + let resolved_callable = bound_method.map_or(callable, |bound| bound.function); + let Some((w_code, nparams, has_closure)) = + (unsafe { resolve_inlinable_callee(resolved_callable) }) else { if std::env::var_os("PYRE_FBW_INLINE_DIAG").is_some() { eprintln!("[inline-decline] pc={} callee not inlinable", op.pc); @@ -1557,14 +1643,20 @@ pub(crate) fn try_walker_inline_user_call( let (callee_args, callee_arg_concretes) = if is_call_kw { // A keyword call folds its `kwnames`->parameter permutation at trace // time (`fbw_reorder_call_kw_args`) so the reordered param-order args - // seed the callee exactly like a positional call. A bound-method-form - // keyword call is not yet folded. - if method_form { - return Ok(None); - } - let Some(reordered) = - (unsafe { fbw_reorder_call_kw_args(r_args, &arg_concretes, w_code, nparams) }) - else { + // seed the callee exactly like a positional call. Method form + // prepends its receiver before that permutation, matching + // `call_kw`'s `Arguments.prepend` / `funcrun_obj` path. + let receiver = if let Some(bound) = bound_method { + // Placeholder until the resolved half reads Method.w_self live. + Some((bound.method_op, ConcreteValue::Ref(bound.receiver))) + } else if method_form { + Some((r_args[1], arg_concretes[1])) + } else { + None + }; + let Some(reordered) = (unsafe { + fbw_reorder_call_kw_args(r_args, &arg_concretes, w_code, nparams, receiver) + }) else { return Ok(None); }; reordered @@ -1590,7 +1682,12 @@ pub(crate) fn try_walker_inline_user_call( } else { let mut callee_args = Vec::with_capacity(r_args.len().saturating_sub(1)); let mut callee_arg_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); - if method_form { + if let Some(bound) = bound_method { + // Placeholder until the non-emitting eligibility checks finish; + // the resolved half replaces it with GetfieldGcR(Method.w_self). + callee_args.push(bound.method_op); + callee_arg_concretes.push(ConcreteValue::Ref(bound.receiver)); + } else if method_form { callee_args.push(r_args[1]); callee_arg_concretes.push(arg_concretes[1]); } @@ -1607,13 +1704,14 @@ pub(crate) fn try_walker_inline_user_call( call_descr, dst_bank, dst, - callable, + resolved_callable, r_args[0], - callable, + resolved_callable, arg_concretes, callee_args, callee_arg_concretes, method_form, + bound_method, w_code, nparams, has_closure, @@ -1811,6 +1909,400 @@ pub(crate) fn walker_ec_leave( ctx.opimpl_virtual_ref_finish(callee_frame); } +/// Resolve the generated builtin-wrapper argument slice's array-item +/// descriptor. The first instruction's arraylen descriptor is deliberately +/// not interchangeable with the later getarrayitem descriptor. +pub(super) fn wrapper_args_item_descr_index(code: &[u8]) -> Option { + let mut wrapper_ops = crate::jitcode_runtime::decoded_ops(code); + let wrapper_abi_matches = wrapper_ops.next().is_some_and(|first| { + first.key == "arraylen_gc/rd>i" && code.get(first.pc + 1).copied() == Some(0) + }); + if !wrapper_abi_matches { + return None; + } + crate::jitcode_runtime::decoded_ops(code) + .find(|decoded| { + decoded.key == "getarrayitem_gc_r/rid>r" && code.get(decoded.pc + 1).copied() == Some(0) + }) + .and_then(|decoded| { + let lo = *code.get(decoded.pc + 3)? as usize; + let hi = *code.get(decoded.pc + 4)? as usize; + let pool_index = lo | (hi << 8); + crate::jitcode_runtime::all_descr_refs() + .get(pool_index) + .map(|descr| descr.index()) + }) +} + +/// `BuiltinCode.func` is an RPython PBC: the codewriter turns its finite +/// target family into an indirect call whose address is resolved back to the +/// generated target JitCode by `MetaInterpStaticData.bytecode_for_address` +/// (`pyjitpl.py:2174-2186`). The interpreter-level `call_fn` helper hides +/// that indirect call behind `Function -> BuiltinCode -> func`, so recover +/// the same target here and enter the generated wrapper with its one red +/// `&[PyObjectRef]` argument. +/// +/// The slice is represented to the translated body as a GC array. Build that +/// array in trace IR and seed its heap-cache entries from the live CALL +/// operands; this preserves a distinct red receiver for every method call. +/// In particular, a bound Method's receiver is read from its immutable +/// `w_self` field rather than baked from the recording-time object. +pub(crate) fn try_walker_inline_builtin_call( + ctx: &mut WalkContext<'_, '_, Sym>, + op: &DecodedOp, + code: &[u8], + ref_operand_offset: usize, + r_args: &[OpRef], + pyre_helper: majit_ir::PyreHelperKind, + dst_bank: char, + dst: usize, +) -> Result, DispatchError> { + if !ctx.is_authoritative_executor + || ctx.fbw_mode.inline_subwalk + || !matches!( + pyre_helper, + majit_ir::PyreHelperKind::CallFn | majit_ir::PyreHelperKind::CallKw + ) + || r_args.len() < 2 + || dst_bank != 'r' + { + return Ok(None); + } + + let mut arg_concretes = read_ref_var_list_concrete(code, op, ref_operand_offset, ctx); + for i in 0..2 { + if matches!(arg_concretes.get(i), Some(ConcreteValue::Null)) { + if let Some(majit_ir::Value::Ref(r)) = ctx.trace_ctx.box_value(r_args[i]) { + if r != majit_ir::GcRef::NO_CONCRETE && r.as_usize() != 0 { + arg_concretes[i] = ConcreteValue::Ref(r.as_usize() as pyre_object::PyObjectRef); + } + } + } + } + let ConcreteValue::Ref(callable_operand) = arg_concretes[0] else { + return Ok(None); + }; + if callable_operand.is_null() { + return Ok(None); + } + let null_or_self = match arg_concretes[1] { + ConcreteValue::Ref(value) => value, + ConcreteValue::Null => pyre_object::PY_NULL, + _ => return Ok(None), + }; + let method_form = !null_or_self.is_null() && null_or_self != pyre_object::PY_NULL; + let bound_method = !method_form && unsafe { pyre_object::is_method(callable_operand) }; + let (callable, receiver) = if bound_method { + let function = unsafe { pyre_object::w_method_get_func(callable_operand) }; + let receiver = unsafe { pyre_object::w_method_get_self(callable_operand) }; + if function.is_null() || receiver.is_null() { + return Ok(None); + } + (function, Some(receiver)) + } else { + (callable_operand, method_form.then_some(null_or_self)) + }; + if !unsafe { pyre_interpreter::is_function(callable) } { + return Ok(None); + } + let builtin_code = + unsafe { pyre_interpreter::function_get_code(callable) } as pyre_object::PyObjectRef; + if builtin_code.is_null() || !unsafe { pyre_interpreter::is_builtin_code(builtin_code) } { + return Ok(None); + } + let fnaddr = unsafe { pyre_interpreter::builtin_code_get(builtin_code) as usize }; + let Some(jitcode) = crate::state::bytecode_for_address(fnaddr) else { + return Ok(None); + }; + let Some(body) = crate::jitcode_dispatch::sub_jitcode_body_by_index(jitcode.index()) else { + return Ok(None); + }; + if body.num_regs_r < 1 { + return Ok(None); + } + // Guards inside the generated wrapper must resume at the outer Python + // CALL, because helper JitCodes have no blackhole entry point of their + // own. The full-body symbol is the authority for that caller frame's + // liveness and resume coordinate (the same setup used by the orthodox + // w_list_append descent below). Resolve it before recording any guards + // or synthetic allocations so a missing coordinate is a clean decline. + let sym_ptr = ctx.fbw_mode.snapshot_sym; + if sym_ptr.is_null() { + return Ok(None); + } + // SAFETY: snapshot_sym is installed for the lifetime of the enclosing + // full-body walk and is read-only here. + let sym = unsafe { &*sym_ptr }; + if sym.jitcode().is_null() { + return Ok(None); + } + let (call_site_py_pc, vsd_value, outer_jitcode_index, call_site_marker) = unsafe { + let jc = &*sym.jitcode(); + let jc_index = jc.index as u32; + let marker = jc.payload.resume_marker_for_jitcode_pc(op.pc); + let mut py = python_pc_for_jitcode_pc(&jc.payload.metadata, op.pc); + if jc.payload.code_ptr.is_null() { + (py, sym.valuestackdepth() as i64, jc_index, marker) + } else { + let codeobj = &*jc.payload.code_ptr; + py = skip_python_trivia_forward(codeobj, py as usize) as u32; + let depth = if jc.payload.depth_trivia_populated() { + jc.payload.depth_trivia_for_jitcode_pc(op.pc) + } else { + crate::liveness::liveness_for(jc.payload.code_ptr) + .depth_at_py_pc() + .get(py as usize) + .copied() + }; + let vsd = depth + .map(|d| (sym.nlocals() + d as usize) as i64) + .unwrap_or(sym.valuestackdepth() as i64); + (py, vsd, jc_index, marker) + } + }; + let call_site_word = call_site_marker + .map(|marker| marker as i32) + .unwrap_or(majit_ir::resumedata::NO_JITCODE_PC); + // Rewind point for the un-lowered-helper decline below. Nothing above + // this line records IR or touches the heap cache, so cutting back to it + // leaves the caller's trace exactly as the ordinary residual call found it. + let pre_fold_pos = ctx.trace_ctx.get_trace_position(); + let call_site_active = collect_outer_active_boxes( + sym, + ctx.trace_ctx, + ctx.registers_i, + ctx.registers_r, + ctx.registers_f, + outer_jitcode_index, + false, + call_site_word, + op.pc as i32, + OuterActiveBoxesEntryTwin::Plain, + "builtin_wrapper_call_site", + None, + &[], + ); + + // The generated builtin-wrapper ABI takes its `&[PyObjectRef]` argument + // in r0 and begins by checking its length. Resolve that instruction's + // descriptor operand now, before switching the sub-walk to the global + // descriptor pool below. The wrapper starts with arraylen(r0), but Charon + // emits a distinct descriptor for slice length (header metadata) and + // slice item access (element metadata). Heapcache array-item keys use the + // latter, exactly like RPython `_do_getarrayitem_gc_any(arraydescr)`; + // seeding under the arraylen descriptor makes the later getitem miss and + // manufactures a Box without its recording-time `.value`. + let Some(wrapper_args_descr_index) = wrapper_args_item_descr_index(body.code) else { + return Ok(None); + }; + + let mut callable_guard_op = r_args[0]; + let mut receiver_op = method_form.then_some(r_args[1]); + if bound_method { + // pypy/interpreter/function.py `_Method._immutable_fields_`: + // guard the carrier layout, read both fields live, and only promote + // the immutable function identity used to select BuiltinCode.func. + let method_type_addr = &pyre_object::function::METHOD_TYPE as *const _ as i64; + walker_guard_class(ctx, op.pc, r_args[0], method_type_addr)?; + callable_guard_op = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + r_args[0], + crate::descr::method_w_function_descr(), + ); + let live_receiver = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + r_args[0], + crate::descr::method_w_self_descr(), + ); + ctx.trace_ctx.try_set_opref_concrete( + live_receiver, + majit_ir::Value::Ref(majit_ir::GcRef(receiver.unwrap() as usize)), + ); + receiver_op = Some(live_receiver); + } + if !callable_guard_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(callable as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[callable_guard_op, expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_guard_op, expected); + } + + let mut wrapper_items = Vec::with_capacity(r_args.len().saturating_sub(1)); + let mut wrapper_item_concretes = Vec::with_capacity(arg_concretes.len().saturating_sub(1)); + if let Some(receiver_op) = receiver_op { + wrapper_items.push(receiver_op); + wrapper_item_concretes.push(ConcreteValue::Ref(receiver.unwrap())); + } + wrapper_items.extend_from_slice(&r_args[2..]); + wrapper_item_concretes.extend_from_slice(&arg_concretes[2..]); + for (&item, concrete) in wrapper_items.iter().zip(&wrapper_item_concretes) { + if let ConcreteValue::Ref(value) = concrete + && !value.is_null() + { + // Box.value is the recording-time shadow, not a compile-time + // constant. The generated wrapper's getarrayitem returns this + // same live box; seeding it lets py_type_check choose its observed + // arm and emit the corresponding guards while retaining `self` + // as a red input. + ctx.trace_ctx.try_set_opref_concrete( + item, + majit_ir::Value::Ref(majit_ir::GcRef(*value as usize)), + ); + } + } + + let array_descr = crate::state::pyobject_gcarray_descr(); + let len = ctx.trace_ctx.const_int(wrapper_items.len() as i64); + let args_array = + ctx.trace_ctx + .record_op_with_descr(OpCode::NewArrayClear, &[len], array_descr.clone()); + ctx.trace_ctx + .heap_cache_mut() + .new_array(args_array, len, true); + for (index, &item) in wrapper_items.iter().enumerate() { + let index = ctx.trace_ctx.const_int(index as i64); + ctx.trace_ctx.record_op_with_descr( + OpCode::SetarrayitemGc, + &[args_array, index, item], + array_descr.clone(), + ); + ctx.trace_ctx + .heapcache_setarrayitem(args_array, index, wrapper_args_descr_index, item); + } + + if sym.owns_virtualizable_shadow() { + let last_instr = call_site_py_pc as i64 - 1; + let last_instr_op = ctx.trace_ctx.const_int(last_instr); + crate::trace_opcode::mirror_vable_static_to_boxes( + ctx.trace_ctx, + "last_instr", + last_instr_op, + Value::Int(last_instr), + ); + let vsd_op = ctx.trace_ctx.const_int(vsd_value); + crate::trace_opcode::mirror_vable_static_to_boxes( + ctx.trace_ctx, + "valuestackdepth", + vsd_op, + Value::Int(vsd_value), + ); + } + + // Build-time canonical helper JitCodes use the one global Assembler + // descriptor pool. Temporarily give the wrapper sub-frame that pool; + // its nested inline_call descriptors then resolve the generated child + // JitCodes (e.g. W_Random::random -> Random::random) by global index. + let saved_entry = ctx.entry_py_pc; + let saved_marker = ctx.outer_resume_marker_jit_pc; + let saved_oji = ctx.outer_jitcode_index; + let saved_active = std::mem::take(&mut ctx.outer_active_boxes); + let saved_descr_refs = ctx.descr_refs; + let saved_raw_descrs = ctx.raw_descrs; + let saved_lookup = ctx.sub_jitcode_lookup; + let saved_fbw_mode = ctx.fbw_mode; + let journal_before = fbw_store_journal_len(); + let unjournaled_before = fbw_has_unjournaled_effect(); + ctx.entry_py_pc = EntryPyPc::Jit(op.pc); + ctx.outer_resume_marker_jit_pc = call_site_marker; + ctx.outer_jitcode_index = outer_jitcode_index; + ctx.outer_active_boxes = call_site_active; + ctx.descr_refs = crate::jitcode_runtime::all_descr_refs(); + ctx.raw_descrs = RawDescrPool::Global; + ctx.sub_jitcode_lookup = &GLOBAL_SUB_JITCODE_LOOKUP_FN; + ctx.fbw_mode.inline_subwalk = true; + let walk_result = run_sub_jitcode_walk( + ctx, + op.pc, + &body, + &[], + &[], + &[args_array], + &[ConcreteValue::Null], + &[], + ); + ctx.fbw_mode = saved_fbw_mode; + ctx.entry_py_pc = saved_entry; + ctx.outer_resume_marker_jit_pc = saved_marker; + ctx.outer_jitcode_index = saved_oji; + ctx.outer_active_boxes = saved_active; + ctx.descr_refs = saved_descr_refs; + ctx.raw_descrs = saved_raw_descrs; + ctx.sub_jitcode_lookup = saved_lookup; + + let walk_result = match walk_result { + Ok(outcome) => outcome, + // `try_execute_residual_call_via_executor` declines an un-lowered + // in-body helper (a `>>47` symbolic fnaddr) while inlining a + // sub-jitcode, so the descent aborts instead of baking the hash as a + // code address. Propagating that abort from here strands the CALL: + // this walk is the authoritative executor and the descent declined + // *before* running the call, so the aborted trace resumes past a + // Python instruction whose effect never happened — `d.popleft()` + // returns its value and leaves the element in place. Roll the partial + // descent back and let the ordinary residual call run, the same way + // the orthodox `w_list_append` descent does. A descent that already + // applied an effect cannot be rewound this way, so it keeps the abort. + Err(DispatchError::OrthodoxSubWalkTraceUnsupported { .. }) + if fbw_store_journal_len() == journal_before + && fbw_has_unjournaled_effect() == unjournaled_before => + { + ctx.trace_ctx.cut_trace(pre_fold_pos); + ctx.trace_ctx.heap_cache_mut().reset(); + bool_box_truth_reset(); + return Ok(None); + } + Err(error) => return Err(error), + }; + match walk_result { + DispatchOutcome::SubReturn { + result: Some(value), + } => { + let concrete = concrete_from_recorded_opref(ctx, value); + write_ref_reg(ctx, op.pc, dst, value, concrete)?; + Ok(Some((DispatchOutcome::Continue, op.next_pc))) + } + DispatchOutcome::SubReturn { result: None } => { + Err(DispatchError::UnexpectedVoidSubReturn { pc: op.pc }) + } + DispatchOutcome::SubRaise { exc, exc_concrete } => { + if let Some(target) = try_catch_exception_at(code, op.next_pc) { + ctx.last_exc_value = Some(exc); + ctx.last_exc_value_concrete = exc_concrete; + Ok(Some((DispatchOutcome::Continue, target))) + } else { + Ok(Some(( + DispatchOutcome::SubRaise { exc, exc_concrete }, + op.next_pc, + ))) + } + } + DispatchOutcome::Terminate => Ok(Some((DispatchOutcome::Terminate, op.next_pc))), + DispatchOutcome::SwitchToBlackhole { + reason, + raising_exception, + } => Ok(Some(( + DispatchOutcome::SwitchToBlackhole { + reason, + raising_exception, + }, + op.next_pc, + ))), + DispatchOutcome::CloseLoop { .. } + | DispatchOutcome::CompileTracePending { .. } + | DispatchOutcome::SubLoopCalleeCallAssembler { .. } => { + Err(DispatchError::SubWalkClosedLoop { pc: op.pc }) + } + DispatchOutcome::Continue => { + unreachable!( + "walk() only exits on Terminate / SubReturn / SubRaise / SwitchToBlackhole" + ) + } + } +} + /// Shared post-resolution half of the FBW inline lever. Ordinary Python calls /// resolve their callee from the CALL operand; builtin-dispatch specializers /// resolve an app-level descriptor first and enter here with that function as @@ -1829,9 +2321,10 @@ pub(crate) fn try_walker_inline_resolved_user_call( callable_guard_op: OpRef, callable_guard_value: pyre_object::PyObjectRef, arg_concretes: Vec, - callee_args: Vec, - callee_arg_concretes: Vec, + mut callee_args: Vec, + mut callee_arg_concretes: Vec, method_form: bool, + bound_method: Option, w_code: *const (), nparams: usize, has_closure: bool, @@ -1840,11 +2333,66 @@ pub(crate) fn try_walker_inline_resolved_user_call( allow_method_load_attr: bool, require_str_result: bool, ) -> Result, DispatchError> { - // Only exact-positional, closure-free calls: every callee local [0..nparams] - // is bound from a passed arg, none from defaults/varargs/cells. - if has_closure || callee_args.len() != nparams { + // `Function.funccall_valuestack` fills a missing positional tail from + // `defs_w` before entering the frame (`function.py:188-193,217-231`). + // Mirror that frame shape here. Placeholder boxes are replaced by live + // guarded tuple-item reads after all non-emitting eligibility checks. + let positional_defaults = if callee_args.len() < nparams { + let Some(defaults) = + (unsafe { positional_defaults_for_inline(callable, callee_args.len(), nparams) }) + else { + return Ok(None); + }; + for &(_, _, value) in &defaults.values { + callee_args.push(OpRef::NONE); + callee_arg_concretes.push(ConcreteValue::Ref(value)); + } + Some(defaults) + } else { + None + }; + // Vararg/over-arity calls still use the ordinary residual path. A closure + // is admissible when it has freevars only: the existing cell objects can + // be threaded into this callee's own frame exactly as + // PyFrame::finish_for_call_with_globals_obj does. A callee with cellvars + // needs fresh cell allocation and stays residual until that constructor + // half is ported too. + if callee_args.len() != nparams { return Ok(None); } + let raw_callee_code = unsafe { + pyre_interpreter::w_code_get_ptr(w_code as pyre_object::PyObjectRef) + as *const pyre_interpreter::CodeObject + }; + if raw_callee_code.is_null() { + return Ok(None); + } + let callee_code = unsafe { &*raw_callee_code }; + let mut concrete_freevar_cells = Vec::new(); + let concrete_closure = if has_closure { + if !callee_code.cellvars.is_empty() { + return Ok(None); + } + let closure = unsafe { pyre_interpreter::function_get_closure(callable) }; + if closure.is_null() + || !unsafe { pyre_object::is_tuple(closure) } + || unsafe { pyre_object::w_tuple_len(closure) } != callee_code.freevars.len() + { + return Ok(None); + } + for i in 0..callee_code.freevars.len() { + let Some(cell) = (unsafe { pyre_object::w_tuple_getitem(closure, i as i64) }) else { + return Ok(None); + }; + concrete_freevar_cells.push(cell); + } + closure + } else { + if !callee_code.freevars.is_empty() { + return Ok(None); + } + pyre_object::PY_NULL + }; // Bound recursive inlining at `max_unroll_recursion`: a callee already // this deep on the FBW inline stack falls back to a residual call rather // than unrolling its (exponentially branching) call tree at trace time. @@ -1903,34 +2451,31 @@ pub(crate) fn try_walker_inline_resolved_user_call( // will not happen and admits a body whose real residual a replay would // double. // - // The two widths are folded in one pass because the accepted tag set - // depends on which one holds: only the int table covers `And` / `Or` / - // `Xor`, so those need every argument to be an exact plain int, while - // `Add` / `Subtract` / `Multiply` are in both tables and take either. - // `bool` is excluded from both: `is_plain_int1` rejects it, so an - // argument list carrying one stays on the conservative side. - let (args_all_exact_plain_int, args_all_exact_numeric) = - callee_arg_concretes - .iter() - .fold((true, true), |(all_int, all_numeric), concrete| { - let (exact_int, exact_float) = match concrete { - ConcreteValue::Int(_) => (true, false), - ConcreteValue::Float(_) => (false, true), - ConcreteValue::Ref(obj) if !obj.is_null() => unsafe { - ( - pyre_object::is_plain_int1(*obj), - pyre_object::is_plain_float_strict(*obj), - ) - }, - ConcreteValue::Bool(_) | ConcreteValue::Ref(_) | ConcreteValue::Null => { - (false, false) - } - }; - ( - all_int && exact_int, - all_numeric && (exact_int || exact_float), - ) - }); + // Preserve exactness per argument. Method-form calls put a usually + // nonnumeric `self` in slot 0; folding all arguments into one boolean + // incorrectly made that erase the proof for an independent numeric `x`. + let exact_numeric_args: Vec = callee_arg_concretes + .iter() + .map(|concrete| { + let (plain_int, exact_float) = match concrete { + ConcreteValue::Int(_) => (true, false), + ConcreteValue::Float(_) => (false, true), + ConcreteValue::Ref(obj) if !obj.is_null() => unsafe { + ( + pyre_object::is_plain_int1(*obj), + pyre_object::is_plain_float_strict(*obj), + ) + }, + ConcreteValue::Bool(_) | ConcreteValue::Ref(_) | ConcreteValue::Null => { + (false, false) + } + }; + ExactNumericArg { + numeric: plain_int || exact_float, + plain_int, + } + }) + .collect(); let args_all_builtin_integer = callee_arg_concretes.iter().all(|concrete| match concrete { ConcreteValue::Int(_) | ConcreteValue::Bool(_) => true, ConcreteValue::Ref(obj) if !obj.is_null() => unsafe { pyre_object::is_int_or_long(*obj) }, @@ -2008,11 +2553,10 @@ pub(crate) fn try_walker_inline_resolved_user_call( if !bridge_rec_root_selfrec && fbw_hazardous_inline_denied(callee_code_key) { return Ok(None); } - // An inline sub-walk inside a FOR_ITER body resumes a guard at the - // caller's CALL boundary, so deopt re-executes the whole callee. Replaying - // a live-heap mutation would double it; the nested-residual decline catches - // that only after an abort storm. A callee whose body commits nothing - // replays benignly, so admit it. + // A legacy, unseeded inline sub-walk inside a FOR_ITER body resumes a guard + // at the caller's CALL boundary, so deopt re-executes the whole callee. + // Replaying a live-heap mutation would double it, so a Dirty body stays on + // the residual call path. // // A body whose only unproven ops are Python-level CALL residuals is // admitted too: this same gate re-runs for each callee the lever resolves @@ -2022,34 +2566,49 @@ pub(crate) fn try_walker_inline_resolved_user_call( // declines — `helper(i)` calling `add(i, 1, 2)` residualizes both calls // per iteration, though each body on its own is pure arithmetic. let mut foriter_deferred_admit = false; + let mut foriter_dirty_bound = false; if fbw_foriter_inflight_active() { let safety = fbw_callee_body_replay_safety( body.code, - nparams, - args_all_exact_numeric, - args_all_exact_plain_int, + &exact_numeric_args, body.num_regs_i, body.constants_i, body.num_regs_r, body.constants_r, callee_descr_refs, ); - let admit = match safety { + let legacy_admit = match safety { CalleeReplaySafety::Clean => true, CalleeReplaySafety::DeferredCall => { foriter_deferred_admit = !fbw_foriter_deferred_call_denied(callee_code_key); foriter_deferred_admit } - CalleeReplaySafety::Dirty => false, + CalleeReplaySafety::Dirty => { + // A stored bound method has an explicit receiver and can use + // the multi-frame red-frame path below. Keep loop-bearing and + // recursive callees residual: either requires another loop + // header rather than one bounded callee walk. + foriter_dirty_bound = bound_method.is_some() + && !method_form + && !pyre_interpreter::code_has_for_iter(callee_code) + && !pyre_interpreter::code_is_self_recursive(callee_code); + foriter_dirty_bound + } }; if std::env::var_os("PYRE_FBW_INLINE_DIAG").is_some() { eprintln!( - "[inline-foriter-gate] pc={} admit={admit} exact_numeric={args_all_exact_numeric} \ + "[inline-foriter-gate] pc={} legacy_admit={legacy_admit} exact_numeric_args={} \ safety={safety:?} deferred_admit={foriter_deferred_admit}", op.pc, + exact_numeric_args.iter().filter(|arg| arg.numeric).count(), ); } - if !admit { + // A `Dirty` body is not admitted by seeding its frame. Its residual + // can raise, and the local `except` that catches it is a callee-owned + // catch edge the inline path does not compile, so the exception + // escapes the caller instead of being handled where the source + // handles it. Keep the ordinary residual call until that edge exists. + if !legacy_admit { return Ok(None); } } @@ -2189,6 +2748,9 @@ pub(crate) fn try_walker_inline_resolved_user_call( ctx, callee_frame_reg, ); + if foriter_dirty_bound && !try_multiframe { + return Ok(None); + } if !strict_inlinable && !try_multiframe { // A non-self-recursive loop/branch callee that neither the strict nor // the multiframe fast path can serve declines to interpretation @@ -2209,6 +2771,35 @@ pub(crate) fn try_walker_inline_resolved_user_call( return Ok(None); } + let mut callable_guard_op = callable_guard_op; + let mut callable_guard_value = callable_guard_value; + if let Some(bound) = bound_method { + // `_Method._immutable_fields_ = ['w_function', 'w_instance']` + // (pypy/interpreter/function.py:567). Preserve those as red field + // reads: guard only the Method layout and underlying function, then + // pass the live receiver field into the callee. Baking the receiver + // concrete would collapse bound methods with different `self` values. + let method_type_addr = &pyre_object::function::METHOD_TYPE as *const _ as i64; + walker_guard_class(ctx, op.pc, bound.method_op, method_type_addr)?; + let function_op = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + bound.method_op, + crate::descr::method_w_function_descr(), + ); + let receiver_op = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + bound.method_op, + crate::descr::method_w_self_descr(), + ); + ctx.trace_ctx.try_set_opref_concrete( + receiver_op, + majit_ir::Value::Ref(majit_ir::GcRef(bound.receiver as usize)), + ); + callee_args[0] = receiver_op; + callable_guard_op = function_op; + callable_guard_value = bound.function; + } + // Path-1 (#68): the inlined callee's compile-time-constant frame fields, // so a scalar `getfield_vable_r` off its own (unseeded) portal frame — // the `w_globals` namespace for a LOAD_GLOBAL, the promote-to-const @@ -2285,6 +2876,39 @@ pub(crate) fn try_walker_inline_resolved_user_call( walker_capture_snapshot_for_last_guard(ctx, op.pc)?; } + if let Some(defaults) = positional_defaults { + // `defs_w?`: read the live field on every compiled iteration, then + // guard the tuple identity used while tracing. Reassigning + // `f.__defaults__` therefore deopts at the caller's CALL boundary. + let defaults_op = ctx.trace_ctx.record_op_with_descr( + OpCode::GetfieldGcR, + &[callable_guard_op], + crate::descr::function_defs_w_descr(), + ); + ctx.trace_ctx.try_set_opref_concrete( + defaults_op, + majit_ir::Value::Ref(majit_ir::GcRef(defaults.tuple as usize)), + ); + let defaults_expected = ctx.trace_ctx.const_ref(defaults.tuple as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[defaults_op, defaults_expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + + // `defs_w?[*]`: once the field guard pins the tuple, its + // `wrappeditems` pointer and contents are immutable. Preserve the + // actual Ref boxes instead of baking one anchor's concrete value. + let items = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + defaults_op, + crate::descr::tuple_wrappeditems_descr(), + ); + for (param_index, tuple_index, _) in defaults.values { + let index = ctx.trace_ctx.const_int(tuple_index as i64); + callee_args[param_index] = + crate::state::trace_items_block_getitem_value_pure(ctx.trace_ctx, items, index); + } + } + let ( mut callee_regs_r, mut callee_regs_i, @@ -2425,19 +3049,9 @@ pub(crate) fn try_walker_inline_resolved_user_call( // note at that site. if try_multiframe || strict_seed { 'seed: { - // Branch-A frame shape only (mirror REC_CA): no cells. - let raw = unsafe { - pyre_interpreter::w_code_get_ptr(w_code as pyre_object::PyObjectRef) - as *const pyre_interpreter::CodeObject - }; - if raw.is_null() { - if try_multiframe { - return Ok(None); - } - break 'seed; - } - let callee_code = unsafe { &*raw }; - if pyre_interpreter::ncells(callee_code) != 0 { + // Branch-A frame shape only (mirror REC_CA): existing freevar + // cells are admissible, while fresh cellvar allocation is not. + if !callee_code.cellvars.is_empty() { if try_multiframe { return Ok(None); } @@ -2469,23 +3083,26 @@ pub(crate) fn try_walker_inline_resolved_user_call( // `PyTraceback` node on the compiled exception path, not preventing // one. Restore `Ok(None)` here once that node is recorded; it is the // largest single win left in this function. - if (0..callee_code.instructions.len()).any(|pc| { - matches!( - pyre_interpreter::decode_instruction_at(callee_code, pc), - Some(( - pyre_interpreter::bytecode::Instruction::PopJumpIfNone { .. } - | pyre_interpreter::bytecode::Instruction::PopJumpIfNotNone { .. }, - _ - )) - ) - }) { + if (bound_method.is_none() || method_form) + && (0..callee_code.instructions.len()).any(|pc| { + matches!( + pyre_interpreter::decode_instruction_at(callee_code, pc), + Some(( + pyre_interpreter::bytecode::Instruction::PopJumpIfNone { .. } + | pyre_interpreter::bytecode::Instruction::PopJumpIfNotNone { .. }, + _ + )) + ) + }) + { if try_multiframe { return Err(DispatchError::callee_inline_unsupported(op.pc)); } break 'seed; } let nlocals = callee_code.varnames.len(); - let frame_array_size = nlocals + callee_code.max_stackdepth as usize; + let ncells = pyre_interpreter::ncells(callee_code); + let frame_array_size = nlocals + ncells + callee_code.max_stackdepth as usize; let Some(callee_jitcode_index) = crate::state::ensure_jitcode_index(callee_code_key as *const ()) @@ -2531,11 +3148,17 @@ pub(crate) fn try_walker_inline_resolved_user_call( let pycode_const = ctx.trace_ctx.const_ref(w_code as i64); let w_globals_obj_const = ctx.trace_ctx.const_ref(inline_consts.w_globals as i64); let param_boxes: Vec = (0..nparams).map(|i| callee_args[i]).collect(); + let freevar_cells: Vec = concrete_freevar_cells + .iter() + .map(|&cell| ctx.trace_ctx.const_ref(cell as i64)) + .collect(); let callee_frame = crate::helpers::emit_new_pyframe_inline_with_params( ctx.trace_ctx, ¶m_boxes, - frame_array_size, + &freevar_cells, nlocals, + frame_array_size, + nlocals + ncells, pycode_const, w_globals_obj_const, callee_ec, @@ -2569,7 +3192,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( &concrete_args, inline_consts.w_globals as pyre_object::PyObjectRef, concrete_ec, - pyre_object::PY_NULL, + concrete_closure, pyre_interpreter::pyframe::FrameLocalsArrayAllocation::OldGenGc, ), ); @@ -2591,12 +3214,11 @@ pub(crate) fn try_walker_inline_resolved_user_call( // Retain for a possible `SubLoopCalleeCallAssembler` emit. ca_callee_frame = callee_frame; ca_callee_ec = callee_ec; - ca_nlocals = nlocals; + ca_nlocals = nlocals + ncells; ca_concrete_frame = concrete_frame_ptr; callee_frame_seeded = true; } } - // gh#467 forward-flush inputs are captured AT the CALL, after this // iteration's pre-CALL effects and before any callee sub-walk. Hoisting // them above the paused-caller-frame gate lets its try-block decline use @@ -3481,6 +4103,7 @@ pub(crate) fn try_walker_inline_exception_string_override( vec![r_args[2]], vec![ConcreteValue::Ref(concrete_receiver)], true, + None, w_code, nparams, has_closure, @@ -3599,6 +4222,7 @@ pub(crate) fn try_walker_inline_property_get( vec![obj], vec![ConcreteValue::Ref(concrete_obj)], true, + None, w_code, nparams, has_closure, @@ -3699,6 +4323,7 @@ pub(crate) fn try_walker_inline_property_set( ConcreteValue::Ref(concrete_value), ], true, + None, w_code, nparams, has_closure, @@ -3709,9 +4334,52 @@ pub(crate) fn try_walker_inline_property_set( ) } -/// Inline a plain Python `__add__` after the numeric BINARY_OP -/// specializations decline. The receiver class and its version tag pin the -/// descriptor lookup, matching `try_dispatch_binary_special`'s forward arm. +/// Forward dunder selected by `try_dispatch_binary_special` for a non-inplace +/// BINARY_OP. In-place operators have a distinct `__i*__` then binary fallback +/// protocol and therefore stay on the generic path until that protocol is +/// ported as a unit. +pub(super) fn user_binop_forward_dunder( + op: pyre_interpreter::bytecode::BinaryOperator, +) -> Option<&'static str> { + use pyre_interpreter::bytecode::BinaryOperator; + + match op { + BinaryOperator::Add => Some("__add__"), + BinaryOperator::And => Some("__and__"), + BinaryOperator::FloorDivide => Some("__floordiv__"), + BinaryOperator::Lshift => Some("__lshift__"), + BinaryOperator::MatrixMultiply => Some("__matmul__"), + BinaryOperator::Multiply => Some("__mul__"), + BinaryOperator::Or => Some("__or__"), + BinaryOperator::Power => Some("__pow__"), + BinaryOperator::Remainder => Some("__mod__"), + BinaryOperator::Rshift => Some("__rshift__"), + BinaryOperator::Subtract => Some("__sub__"), + BinaryOperator::TrueDivide => Some("__truediv__"), + BinaryOperator::Xor => Some("__xor__"), + BinaryOperator::Subscr + | BinaryOperator::InplaceAdd + | BinaryOperator::InplaceAnd + | BinaryOperator::InplaceFloorDivide + | BinaryOperator::InplaceLshift + | BinaryOperator::InplaceMatrixMultiply + | BinaryOperator::InplaceMultiply + | BinaryOperator::InplaceOr + | BinaryOperator::InplacePower + | BinaryOperator::InplaceRemainder + | BinaryOperator::InplaceRshift + | BinaryOperator::InplaceSubtract + | BinaryOperator::InplaceTrueDivide + | BinaryOperator::InplaceXor => None, + } +} + +/// Inline a plain Python forward arithmetic dunder after the exact numeric +/// BINARY_OP specializations decline. The receiver class and its version tag +/// pin the descriptor lookup, matching `try_dispatch_binary_special`'s +/// forward arm. A proper-subclass rhs still declines below so reflected-method +/// priority is preserved; a traced `NotImplemented` result guards and deopts +/// to the generic dispatcher. #[allow(clippy::too_many_arguments)] pub(crate) fn try_walker_inline_user_binop( ctx: &mut WalkContext<'_, '_, Sym>, @@ -3727,12 +4395,12 @@ pub(crate) fn try_walker_inline_user_binop( return Ok(None); } - let Some(pyre_interpreter::bytecode::BinaryOperator::Add) = - pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag) - else { + let Some(op_kind) = pyre_interpreter::runtime_ops::binary_op_from_tag(op_tag) else { + return Ok(None); + }; + let Some(dunder) = user_binop_forward_dunder(op_kind) else { return Ok(None); }; - let dunder = "__add__"; let lhs = r_args[0]; let rhs = r_args[1]; @@ -3810,6 +4478,7 @@ pub(crate) fn try_walker_inline_user_binop( ConcreteValue::Ref(concrete_rhs), ], true, + None, w_code, nparams, has_closure, @@ -3954,6 +4623,7 @@ pub(crate) fn try_walker_inline_user_compareop( ConcreteValue::Ref(concrete_rhs), ], true, + None, w_code, nparams, has_closure, @@ -4218,8 +4888,9 @@ pub(crate) fn dispatch_inline_call_dr_kind( let (args, arg_width) = read_ref_var_list(code, op, 2, ctx)?; let arg_concretes = read_ref_var_list_concrete(code, op, 2, ctx); - let callee_outcome = - run_sub_jitcode_walk(ctx, op.pc, &sub_body, &[], &[], &args, &arg_concretes, &[])?; + let callee_result = + run_sub_jitcode_walk(ctx, op.pc, &sub_body, &[], &[], &args, &arg_concretes, &[]); + let callee_outcome = callee_result?; match callee_outcome { DispatchOutcome::SubReturn { @@ -4553,7 +5224,7 @@ pub(crate) fn dispatch_inline_call_dirf_kind( let ref_arg_concretes = read_ref_var_list_concrete(code, op, 2 + int_width, ctx); let (float_args, float_width) = read_float_var_list(code, op, 2 + int_width + ref_width, ctx)?; - let callee_outcome = run_sub_jitcode_walk( + let callee_result = run_sub_jitcode_walk( ctx, op.pc, &sub_body, @@ -4562,7 +5233,8 @@ pub(crate) fn dispatch_inline_call_dirf_kind( &ref_args, &ref_arg_concretes, &float_args, - )?; + ); + let callee_outcome = callee_result?; match callee_outcome { DispatchOutcome::SubReturn { diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 4c5ede18131..c893d271bb0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -2850,32 +2850,23 @@ pub(crate) fn residual_call_descr_index_in_body(body_code: &[u8], d: &DecodedOp) /// `_pow` but keeps a cold-path residual for nan/inf/negative-base operands. /// - `Subscr`, `MatrixMultiply` (+ in-place) — no arm in either table. /// -/// Both flags describe the callee's INCOMING arguments, so on their own they -/// say nothing about the operands of any particular binop. A body reaches an -/// operand they do not cover through the residuals this scan treats as -/// replay-safe reads: in `def f(x): return G - x` the left operand is whatever -/// `G` names, and a numeric subclass there defines its own `__sub__`, declines -/// the specialization, and leaves behind a live-heap effect a replay would -/// double. `binop_safe_ref_regs` carries the missing proof — the caller sets -/// a register only while it provably holds an immutable builtin — and both -/// operand registers must be in it. +/// The two provenance sets describe the actual operands of each binop. This +/// admits `def f(self, x): return x + 1` when only `x` is numeric, while still +/// rejecting `self + x` and global numeric subclasses with user dunders. pub(crate) fn residual_call_is_specialized_plain_numeric_binop( body_code: &[u8], - args_all_exact_numeric: bool, - args_all_exact_plain_int: bool, - binop_safe_ref_regs: &[bool; u8::MAX as usize + 1], + numeric_ref_regs: &[bool; u8::MAX as usize + 1], + plain_int_ref_regs: &[bool; u8::MAX as usize + 1], d: &DecodedOp, num_regs_i: usize, constants_i: &[i64], callee_descr_refs: &[DescrRef], ) -> bool { - if !args_all_exact_numeric - || !matches!( - d.key, - "residual_call_ir_r/iIRd>r" | "residual_call_ir_i/iIRd>i" | "residual_call_ir_v/iIRd" - ) - || residual_call_helper_kind_in_body(body_code, d, callee_descr_refs) - != Some(majit_ir::PyreHelperKind::BinaryOp) + if !matches!( + d.key, + "residual_call_ir_r/iIRd>r" | "residual_call_ir_i/iIRd>i" | "residual_call_ir_v/iIRd" + ) || residual_call_helper_kind_in_body(body_code, d, callee_descr_refs) + != Some(majit_ir::PyreHelperKind::BinaryOp) { return false; } @@ -2895,7 +2886,7 @@ pub(crate) fn residual_call_is_specialized_plain_numeric_binop( else { return false; }; - if !binop_safe_ref_regs[lhs_reg as usize] || !binop_safe_ref_regs[rhs_reg as usize] { + if !numeric_ref_regs[lhs_reg as usize] || !numeric_ref_regs[rhs_reg as usize] { return false; } // The first I-list item is the BINARY_OP tag. It must be in the callee's @@ -2930,11 +2921,46 @@ pub(crate) fn residual_call_is_specialized_plain_numeric_binop( | BinaryOperator::InplaceAnd | BinaryOperator::InplaceOr | BinaryOperator::InplaceXor, - ) => args_all_exact_plain_int, + ) => plain_int_ref_regs[lhs_reg as usize] && plain_int_ref_regs[rhs_reg as usize], _ => false, } } +pub(crate) fn residual_call_is_specialized_plain_int_binop( + body_code: &[u8], + d: &DecodedOp, + num_regs_i: usize, + constants_i: &[i64], +) -> bool { + let Some(&i_len) = body_code.get(d.pc + 2) else { + return false; + }; + if i_len == 0 { + return false; + } + let Some(&tag_reg) = body_code.get(d.pc + 3) else { + return false; + }; + let Some(&tag) = (tag_reg as usize) + .checked_sub(num_regs_i) + .and_then(|constant_index| constants_i.get(constant_index)) + else { + return false; + }; + use pyre_interpreter::bytecode::BinaryOperator; + matches!( + pyre_interpreter::runtime_ops::binary_op_from_tag(tag), + Some( + BinaryOperator::And + | BinaryOperator::Or + | BinaryOperator::Xor + | BinaryOperator::InplaceAnd + | BinaryOperator::InplaceOr + | BinaryOperator::InplaceXor + ) + ) +} + pub(crate) fn dispatch_residual_call_iRd_kind( code: &[u8], op: &DecodedOp, @@ -2991,6 +3017,15 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // linear `catch_exception/L`. clear_walk_exception(ctx); + // BuiltinCode.func is an indirect PBC target exactly like RPython's + // gateway wrappers. Enter its generated JitCode before considering the + // user-function-only full-body walk below. + if let Some(inlined) = + try_walker_inline_builtin_call(ctx, op, code, 1, &r_args, ei.pyre_helper, dst_bank, dst)? + { + return Ok(inlined); + } + // #62 slice (3c): attempt full-body-walk inline of a user-function call // unconditionally. Eligible exact-positional closure-free // calls sub-walk the callee body in place of the residual; ineligible @@ -3385,6 +3420,28 @@ pub(crate) fn dispatch_residual_call_iRd_kind( } } + // `type(x)` is `space.type(w_obj)` upstream: promote `w_obj.__class__` + // and return `w_obj.getclass(space)`. Lower that directly instead of + // residualizing the builtin type object's full `descr_call`. + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn + && try_walker_specialize_builtin_type(ctx, code, op, &r_args, dst)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + + // Exact `dict.get(identity_key)` follows the promoted strategy-entry + // shape produced by tracing `DictStrategy.getitem`: pin the key-set + // iterator state and read the resolved entry value live. + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn + && try_walker_specialize_builtin_dict_get(ctx, code, op, &r_args, dst)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + // `len(x)` on an exact canonical list: inline the strategy-guarded // length read (guard_value callable + guard_class + exact w_class + // guard_value strategy + length getfield + wrapint) instead of the @@ -3414,6 +3471,34 @@ pub(crate) fn dispatch_residual_call_iRd_kind( { return Ok((DispatchOutcome::Continue, op.next_pc)); } + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn + && try_walker_specialize_math_frexp(ctx, code, op, &r_args, dst)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn + && try_walker_specialize_math_ldexp(ctx, code, op, &r_args, dst)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn + && try_walker_specialize_math_isqrt(ctx, code, op, &r_args, dst)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } + if ctx.is_authoritative_executor + && dst_bank == 'r' + && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn + && try_walker_specialize_int_call(ctx, code, op, &r_args, dst)?.is_some() + { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } if ctx.is_authoritative_executor && dst_bank == 'r' && ei.pyre_helper == majit_ir::PyreHelperKind::CallFn @@ -3946,6 +4031,19 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // arity in the Int list). Share the same user-function inline gate as the // plain Ref-only residual, but read the concrete Ref shadows from the // shifted R-list offset. + if let Some(inlined) = try_walker_inline_builtin_call( + ctx, + op, + code, + 1 + i_width, + &r_args, + ei.pyre_helper, + dst_bank, + dst, + )? { + return Ok(inlined); + } + if let Some(inlined) = try_walker_inline_user_call( ctx, op, diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 9cf36279a5e..389eaf5faf0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -4440,6 +4440,276 @@ pub(crate) fn try_walker_specialize_subscr_tuple( Ok(Some(())) } +/// Builtin `type(x)`: +/// +/// ```python +/// # pypy/objspace/std/objspace.py:441-443 +/// jit.promote(w_obj.__class__) +/// return w_obj.getclass(self) +/// ``` +/// +/// Pyre's generic `bh_call_fn` otherwise enters `type_descr_call_impl`, which +/// performs the complete type-constructor protocol for every loop iteration. +/// Pin the callable and the argument's physical/Python class, then return the +/// promoted class object. The generic-exception representation is declined: +/// its observable class can come from `ExcKind` even when physical type and +/// `w_class` match, so it needs a separate kind guard. +pub(crate) fn try_walker_specialize_builtin_type( + ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], + op: &DecodedOp, + r_args: &[OpRef], + dst: usize, +) -> Result, DispatchError> { + if r_args.len() != 3 { + return Ok(None); + } + let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(concrete_callable), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(obj), + ) = (arg_concretes[0], arg_concretes[1], arg_concretes[2]) + else { + return Ok(None); + }; + if concrete_callable.is_null() || !null_or_self.is_null() || obj.is_null() { + return Ok(None); + } + let builtin_type = pyre_object::get_instantiate(&pyre_object::pyobject::TYPE_TYPE); + if builtin_type.is_null() || !std::ptr::eq(concrete_callable, builtin_type) { + return Ok(None); + } + + let tagged = + pyre_object::tagged_int::CAN_BE_TAGGED && pyre_object::tagged_int::is_tagged_int(obj); + let (physical_type, stored_w_class) = if tagged { + ( + &pyre_object::pyobject::INT_TYPE as *const _ as i64, + pyre_object::get_instantiate(&pyre_object::pyobject::INT_TYPE), + ) + } else { + let physical_type = unsafe { (*obj).ob_type } as i64; + let stored_w_class = unsafe { (*obj).w_class }; + if unsafe { pyre_object::is_exception(obj) } { + let generic_exception = + pyre_object::get_instantiate(&pyre_object::interp_exceptions::EXCEPTION_TYPE); + if stored_w_class.is_null() || std::ptr::eq(stored_w_class, generic_exception) { + return Ok(None); + } + } + (physical_type, stored_w_class) + }; + let Some(result_type) = pyre_interpreter::typedef::r#type(obj) else { + return Ok(None); + }; + let result_type = result_type.as_ptr(); + + let callable_op = r_args[0]; + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(concrete_callable as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[callable_op, expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + + let obj_op = r_args[2]; + walker_guard_class(ctx, op.pc, obj_op, physical_type)?; + // A tagged int has no dereferenceable `w_class` field; GuardClass's boxed + // leg is enough because both representations return the canonical int + // class. Every other populated `w_class` is the live promoted field. + if !tagged && !stored_w_class.is_null() { + walker_guard_exact_w_class(ctx, op.pc, obj_op, stored_w_class)?; + } + let result = ctx.trace_ctx.const_ref(result_type as i64); + write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', result)?; + Ok(Some(())) +} + +fn is_builtin_dict_get_function(callable: pyre_object::PyObjectRef) -> bool { + if callable.is_null() || !unsafe { pyre_interpreter::is_function(callable) } { + return false; + } + let code = unsafe { pyre_interpreter::function_get_code(callable) } as pyre_object::PyObjectRef; + !code.is_null() + && unsafe { pyre_interpreter::is_builtin_code(code) } + && unsafe { pyre_interpreter::builtin_code_get(code) as usize } + == pyre_interpreter::type_methods::dict_method_get as *const () as usize +} + +/// `dict.get` on an exact dictionary and an identity-present promoted key. +/// +/// PyPy traces `DictStrategy.getitem` into its r_dict lookup. Once the +/// observed key is the exact stored key, the hash/equality probe selects a +/// stable entry index; PyPy's native iterator/table state keeps that selection +/// valid until the key set changes. Pyre represents the same state explicitly +/// as `W_DictObject.keys_version`, so guard it and issue a live nth-value read. +/// Value-only replacement is intentionally visible without invalidation. +/// +/// Equal-but-nonidentical keys and misses remain residual because reproducing +/// their hash/equality effects requires the complete r_dict probe. +pub(crate) fn try_walker_specialize_builtin_dict_get( + ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], + op: &DecodedOp, + r_args: &[OpRef], + dst: usize, +) -> Result, DispatchError> { + if !(r_args.len() == 3 || r_args.len() == 4) { + return Ok(None); + } + let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(callable_operand), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(key), + ) = (arg_concretes[0], arg_concretes[1], arg_concretes[2]) + else { + return Ok(None); + }; + if callable_operand.is_null() || key.is_null() { + return Ok(None); + } + // LOAD_ATTR's generic method path produces `[Method, PY_NULL, key]`; + // LOAD_METHOD's split path produces `[Function, receiver, key]`. + // `_Method._immutable_fields_` lets both converge on the same live + // function/receiver reads. + let bound_method = + null_or_self.is_null() && unsafe { pyre_object::is_method(callable_operand) }; + let (callable, dict) = if bound_method { + ( + unsafe { pyre_object::w_method_get_func(callable_operand) }, + unsafe { pyre_object::w_method_get_self(callable_operand) }, + ) + } else { + (callable_operand, null_or_self) + }; + if callable.is_null() || dict.is_null() || !is_builtin_dict_get_function(callable) { + return Ok(None); + } + let canonical_dict = pyre_object::get_instantiate(&pyre_object::pyobject::DICT_TYPE); + if canonical_dict.is_null() + || !unsafe { + std::ptr::eq((*dict).ob_type, &pyre_object::pyobject::DICT_TYPE) + && std::ptr::eq((*dict).w_class, canonical_dict) + } + { + return Ok(None); + } + + let len = unsafe { pyre_object::w_dict_len(dict) }; + let mut found = None; + for index in 0..len { + let Some((stored_key, value)) = + (unsafe { pyre_object::dictmultiobject::w_dict_nth_item(dict, index) }) + else { + return Ok(None); + }; + if std::ptr::eq(stored_key, key) { + found = Some((index, value)); + break; + } + } + let Some((index, concrete_value)) = found else { + return Ok(None); + }; + let concrete_version = unsafe { pyre_object::dictmultiobject::w_dict_keys_version(dict) }; + + let mut callable_op = r_args[0]; + let dict_op; + if bound_method { + walker_guard_class( + ctx, + op.pc, + r_args[0], + &pyre_object::function::METHOD_TYPE as *const _ as i64, + )?; + callable_op = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + r_args[0], + crate::descr::method_w_function_descr(), + ); + dict_op = crate::state::opimpl_getfield_gc_r( + ctx.trace_ctx, + r_args[0], + crate::descr::method_w_self_descr(), + ); + ctx.trace_ctx.try_set_opref_concrete( + dict_op, + majit_ir::Value::Ref(majit_ir::GcRef(dict as usize)), + ); + } else { + dict_op = r_args[1]; + } + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(callable as i64); + walker_emit_fold_guard_with_snapshot( + ctx, + op.pc, + OpCode::GuardValue, + &[callable_op, expected], + )?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + walker_guard_class( + ctx, + op.pc, + dict_op, + &pyre_object::pyobject::DICT_TYPE as *const _ as i64, + )?; + walker_guard_exact_w_class(ctx, op.pc, dict_op, canonical_dict)?; + let key_op = r_args[2]; + let key_expected = ctx.trace_ctx.const_ref(key as i64); + if !key_op.is_constant() { + walker_emit_fold_guard_with_snapshot( + ctx, + op.pc, + OpCode::GuardValue, + &[key_op, key_expected], + )?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(key_op, key_expected); + } + let live_version = crate::state::opimpl_getfield_gc_i( + ctx.trace_ctx, + dict_op, + crate::descr::dict_keys_version_descr(), + ); + let expected_version = ctx.trace_ctx.const_int(concrete_version as i64); + walker_emit_fold_guard_with_snapshot( + ctx, + op.pc, + OpCode::GuardValue, + &[live_version, expected_version], + )?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(live_version, expected_version); + + let index_op = ctx.trace_ctx.const_int(index as i64); + let value = ctx.trace_ctx.call_ref_typed_with_effect( + crate::helpers::jit_dict_nth_value as *const (), + &[dict_op, index_op], + &[majit_ir::Type::Ref, majit_ir::Type::Int], + majit_ir::EffectInfo::new( + majit_ir::ExtraEffect::CannotRaise, + majit_ir::OopSpecIndex::None, + ), + ); + ctx.trace_ctx.set_opref_concrete( + value, + majit_ir::Value::Ref(majit_ir::GcRef(concrete_value as usize)), + ); + write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', value)?; + Ok(Some(())) +} + /// `len(x)` on an exact canonical `W_ListObject` / `W_UnicodeObject` / /// `W_TupleObject`: /// lower the opaque `bh_call_fn(len_builtin, PY_NULL, x)` residual to the @@ -4753,6 +5023,433 @@ pub(crate) fn try_walker_specialize_math_sqrt( Ok(Some(())) } +/// `math.frexp(x)` on an exact int/float argument. RPython lowers +/// `ll_math_frexp` to two unboxed results and `space.newtuple2`; emit the same +/// shape as two pure typed calls followed by a virtualizable object tuple. +/// This avoids the opaque builtin dispatch and the concrete tuple/element +/// allocations in numeric loops. Rebound callables, subclasses, and other +/// coercion shapes retain the generic residual path. +pub(crate) fn try_walker_specialize_math_frexp( + ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], + op: &DecodedOp, + r_args: &[OpRef], + dst: usize, +) -> Result, DispatchError> { + if r_args.len() != 3 { + return Ok(None); + } + let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(concrete_callable), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(arg_obj), + ) = (arg_concretes[0], arg_concretes[1], arg_concretes[2]) + else { + return Ok(None); + }; + if concrete_callable.is_null() || !null_or_self.is_null() || arg_obj.is_null() { + return Ok(None); + } + if !pyre_interpreter::module::math::interp_math::is_math_frexp_function(concrete_callable) { + return Ok(None); + } + let (is_int, x_value) = unsafe { + if !pyre_object::is_exact_builtin_instance(arg_obj) { + return Ok(None); + } + if pyre_object::is_int(arg_obj) { + (true, pyre_object::w_int_get_value(arg_obj) as f64) + } else if pyre_object::is_float(arg_obj) { + (false, pyre_object::w_float_get_value(arg_obj)) + } else { + return Ok(None); + } + }; + + // Execute the skipped builtin once so all concrete tuple/boxing choices + // match the interpreter exactly. + let boxed_result = { + let _plain_guard = pyre_interpreter::call::force_plain_eval(); + pyre_interpreter::call::call_function_impl_result(concrete_callable, &[arg_obj]) + }; + let Ok(boxed_result) = boxed_result else { + return Ok(None); + }; + let (Some(mantissa_obj), Some(exponent_obj)) = (unsafe { + ( + pyre_object::w_tuple_getitem(boxed_result, 0), + pyre_object::w_tuple_getitem(boxed_result, 1), + ) + }) else { + return Ok(None); + }; + let mantissa_value = unsafe { pyre_object::w_float_get_value(mantissa_obj) }; + let exponent_value = unsafe { pyre_object::w_int_get_value(exponent_obj) }; + + let callable_op = r_args[0]; + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(concrete_callable as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[callable_op, expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + let x = walker_coerce_operand_to_float(ctx, op.pc, r_args[2], arg_obj, is_int, x_value, false)?; + let mantissa = ctx.trace_ctx.call_float_typed_with_effect( + pyre_interpreter::module::math::interp_math::jit_math_frexp_mantissa as *const (), + &[x], + &[majit_ir::Type::Float], + majit_metainterp::CANNOT_RAISE_NO_HEAP_EFFECT_INFO, + ); + ctx.trace_ctx + .set_opref_concrete(mantissa, majit_ir::Value::Float(mantissa_value)); + let exponent = ctx.trace_ctx.call_int_typed_with_effect( + pyre_interpreter::module::math::interp_math::jit_math_frexp_exponent as *const (), + &[x], + &[majit_ir::Type::Float], + majit_metainterp::CANNOT_RAISE_NO_HEAP_EFFECT_INFO, + ); + ctx.trace_ctx + .set_opref_concrete(exponent, majit_ir::Value::Int(exponent_value)); + + let mantissa_box = crate::state::wrapfloat(ctx.trace_ctx, mantissa); + ctx.trace_ctx.set_opref_concrete( + mantissa_box, + majit_ir::Value::Ref(majit_ir::GcRef(mantissa_obj as usize)), + ); + let exponent_box = walker_box_int(ctx, op.pc, exponent, exponent_value)?; + ctx.trace_ctx.set_opref_concrete( + exponent_box, + box_int_concrete(exponent_value, exponent_obj as i64), + ); + // `space.newtuple2` selects `W_SpecialisedTupleObject_oo` for the + // float/int pair. The traced allocation must use that same layout: + // UNPACK_SEQUENCE specializes from the record-time concrete object's + // class and reads the two inline `value*` fields. + let tuple = + crate::helpers::emit_specialised_tuple_oo_inline(ctx.trace_ctx, mantissa_box, exponent_box); + ctx.trace_ctx.set_opref_concrete( + tuple, + majit_ir::Value::Ref(majit_ir::GcRef(boxed_result as usize)), + ); + write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', tuple)?; + Ok(Some(())) +} + +/// `math.ldexp(x, exp)` on exact numeric arguments. This is the direct +/// walker equivalent of RPython's `ll_math_ldexp`: carry `x` and `exp` +/// unboxed, call the platform operation, and guard a finite result so the +/// overflow direction resumes in the builtin and raises `OverflowError`. +/// Underflow to signed zero remains on the fast path. Non-finite concrete +/// inputs and non-int exponents retain the generic residual path. +pub(crate) fn try_walker_specialize_math_ldexp( + ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], + op: &DecodedOp, + r_args: &[OpRef], + dst: usize, +) -> Result, DispatchError> { + if r_args.len() != 4 { + return Ok(None); + } + let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(concrete_callable), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(x_obj), + ConcreteValue::Ref(exp_obj), + ) = ( + arg_concretes[0], + arg_concretes[1], + arg_concretes[2], + arg_concretes[3], + ) + else { + return Ok(None); + }; + if concrete_callable.is_null() + || !null_or_self.is_null() + || x_obj.is_null() + || exp_obj.is_null() + || !pyre_interpreter::module::math::interp_math::is_math_ldexp_function(concrete_callable) + { + return Ok(None); + } + let (x_is_int, x_value, exp_value) = unsafe { + if !pyre_object::is_exact_builtin_instance(x_obj) + || !pyre_object::is_exact_builtin_instance(exp_obj) + { + return Ok(None); + } + let (x_is_int, x_value) = if pyre_object::is_int(x_obj) { + (true, pyre_object::w_int_get_value(x_obj) as f64) + } else if pyre_object::is_float(x_obj) { + (false, pyre_object::w_float_get_value(x_obj)) + } else { + return Ok(None); + }; + if !pyre_object::is_int(exp_obj) { + return Ok(None); + } + (x_is_int, x_value, pyre_object::w_int_get_value(exp_obj)) + }; + // The finite-result guard below represents RPython's errno/overflow + // branch. Trace non-finite inputs through the ordinary builtin because + // ll_math_ldexp returns them unchanged rather than taking that guard. + if !x_value.is_finite() { + return Ok(None); + } + let boxed_result = { + let _plain_guard = pyre_interpreter::call::force_plain_eval(); + pyre_interpreter::call::call_function_impl_result(concrete_callable, &[x_obj, exp_obj]) + }; + let Ok(boxed_result) = boxed_result else { + return Ok(None); + }; + let result_value = unsafe { pyre_object::w_float_get_value(boxed_result) }; + + let callable_op = r_args[0]; + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(concrete_callable as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[callable_op, expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + let x = walker_coerce_operand_to_float(ctx, op.pc, r_args[2], x_obj, x_is_int, x_value, false)?; + let (exp_type_addr, exp_descr) = crate::state::int_or_bool_unbox_type_descr(exp_obj); + let exp = walker_unbox_int_typed(ctx, op.pc, r_args[3], exp_type_addr, exp_descr)?; + ctx.trace_ctx + .set_opref_concrete(exp, majit_ir::Value::Int(exp_value)); + let raw = ctx.trace_ctx.call_float_typed_with_effect( + pyre_interpreter::module::math::interp_math::jit_math_ldexp_raw as *const (), + &[x, exp], + &[majit_ir::Type::Float, majit_ir::Type::Int], + majit_metainterp::CANNOT_RAISE_NO_HEAP_EFFECT_INFO, + ); + ctx.trace_ctx + .set_opref_concrete(raw, majit_ir::Value::Float(result_value)); + // `result - result == 0` exactly for every finite value, including + // signed zero; infinity and NaN bail to the raising/propagating builtin. + let diff = ctx.trace_ctx.record_op(OpCode::FloatSub, &[raw, raw]); + ctx.trace_ctx + .set_opref_concrete(diff, majit_ir::Value::Float(0.0)); + let zero = ctx.trace_ctx.const_float(0.0f64.to_bits() as i64); + walker_float_cmp_guard(ctx, op.pc, OpCode::FloatEq, &[diff, zero], true)?; + + let boxed = crate::state::wrapfloat(ctx.trace_ctx, raw); + ctx.trace_ctx.set_opref_concrete( + boxed, + majit_ir::Value::Ref(majit_ir::GcRef(boxed_result as usize)), + ); + write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', boxed)?; + Ok(Some(())) +} + +/// `math.isqrt(n)` on an exact nonnegative machine integer. +/// +/// PyPy exposes `isqrt` from `app_math.py`, so tracing its `W_IntObject` arm +/// carries `n` unboxed through the integer algorithm and virtualizes the +/// result. Pyre's native module wrapper otherwise materializes an `RBigInt` +/// before the walker can see that arm. Recreate the translated shape as an +/// exact-class guard, unbox, pure non-raising integer call, and `wrapint`. +/// Longs, subclasses, negative values, rebound callables, and `__index__` +/// objects retain the generic residual path. +pub(crate) fn try_walker_specialize_math_isqrt( + ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], + op: &DecodedOp, + r_args: &[OpRef], + dst: usize, +) -> Result, DispatchError> { + if r_args.len() != 3 { + return Ok(None); + } + let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(concrete_callable), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(arg_obj), + ) = (arg_concretes[0], arg_concretes[1], arg_concretes[2]) + else { + return Ok(None); + }; + if concrete_callable.is_null() + || !null_or_self.is_null() + || arg_obj.is_null() + || !pyre_interpreter::module::math::interp_math::is_math_isqrt_function(concrete_callable) + { + return Ok(None); + } + let value = unsafe { + if !pyre_object::is_exact_builtin_instance(arg_obj) || !pyre_object::is_int(arg_obj) { + return Ok(None); + } + pyre_object::w_int_get_value(arg_obj) + }; + if value < 0 { + return Ok(None); + } + let boxed_result = { + let _plain_guard = pyre_interpreter::call::force_plain_eval(); + pyre_interpreter::call::call_function_impl_result(concrete_callable, &[arg_obj]) + }; + let Ok(boxed_result) = boxed_result else { + return Ok(None); + }; + if !unsafe { pyre_object::is_int(boxed_result) } { + return Ok(None); + } + let result_value = unsafe { pyre_object::w_int_get_value(boxed_result) }; + + let callable_op = r_args[0]; + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(concrete_callable as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[callable_op, expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + let arg_op = r_args[2]; + let int_type_addr = &pyre_object::pyobject::INT_TYPE as *const _ as i64; + let raw_int = walker_unbox_int(ctx, op.pc, arg_op, int_type_addr)?; + walker_guard_exact_w_class( + ctx, + op.pc, + arg_op, + pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::INT_TYPE), + )?; + ctx.trace_ctx + .set_opref_concrete(raw_int, majit_ir::Value::Int(value)); + let zero = ctx.trace_ctx.const_int(0); + let nonnegative = ctx.trace_ctx.record_op(OpCode::IntGe, &[raw_int, zero]); + ctx.trace_ctx + .set_opref_concrete(nonnegative, majit_ir::Value::Int(1)); + walker_emit_guard_with_snapshot(ctx, op.pc, OpCode::GuardTrue, &[nonnegative])?; + + let raw_result = ctx.trace_ctx.call_int_typed_with_effect( + pyre_interpreter::module::math::interp_math::jit_math_isqrt_i64 as *const (), + &[raw_int], + &[majit_ir::Type::Int], + majit_metainterp::CANNOT_RAISE_NO_HEAP_EFFECT_INFO, + ); + ctx.trace_ctx + .set_opref_concrete(raw_result, majit_ir::Value::Int(result_value)); + let boxed = walker_box_int(ctx, op.pc, raw_result, result_value)?; + ctx.trace_ctx + .set_opref_concrete(boxed, box_int_concrete(result_value, boxed_result as i64)); + write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', boxed)?; + Ok(Some(())) +} + +/// `int(x)` for an exact float whose truncated value fits a machine Signed. +/// +/// PyPy `floatobject.py:newint_from_float` first runs +/// `ovfcheck_float_to_int`; its success arm is exactly +/// `CAST_FLOAT_TO_INT + space.newint`. Emit that arm with the corresponding +/// `-2**63 <= x < 2**63` guards, leaving NaN, infinity, out-of-range values, +/// subclasses, and rebound constructors on the ordinary residual path. The +/// slow arm remains responsible for `newlong_from_float` and its exceptions. +pub(crate) fn try_walker_specialize_int_call( + ctx: &mut WalkContext<'_, '_, Sym>, + code: &[u8], + op: &DecodedOp, + r_args: &[OpRef], + dst: usize, +) -> Result, DispatchError> { + if r_args.len() != 3 { + return Ok(None); + } + let arg_concretes = read_ref_var_list_concrete(code, op, 1, ctx); + let ( + ConcreteValue::Ref(concrete_callable), + ConcreteValue::Ref(null_or_self), + ConcreteValue::Ref(arg_obj), + ) = (arg_concretes[0], arg_concretes[1], arg_concretes[2]) + else { + return Ok(None); + }; + if concrete_callable.is_null() || !null_or_self.is_null() || arg_obj.is_null() { + return Ok(None); + } + let int_type_obj = pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::INT_TYPE); + if !std::ptr::eq(concrete_callable, int_type_obj) { + return Ok(None); + } + let value = unsafe { + if !pyre_object::is_exact_builtin_instance(arg_obj) || !pyre_object::is_float(arg_obj) { + return Ok(None); + } + pyre_object::w_float_get_value(arg_obj) + }; + // `2**63` is exactly representable while `i64::MAX` is not; use a strict + // upper bound, matching ovfcheck_float_to_int on a signed 64-bit target. + const SIGNED_MIN_AS_FLOAT: f64 = -9223372036854775808.0; + const SIGNED_LIMIT_AS_FLOAT: f64 = 9223372036854775808.0; + if !(value >= SIGNED_MIN_AS_FLOAT && value < SIGNED_LIMIT_AS_FLOAT) { + return Ok(None); + } + let boxed_result = { + let _plain_guard = pyre_interpreter::call::force_plain_eval(); + pyre_interpreter::call::call_function_impl_result(concrete_callable, &[arg_obj]) + }; + let Ok(boxed_result) = boxed_result else { + return Ok(None); + }; + if !unsafe { pyre_object::is_int(boxed_result) } { + return Ok(None); + } + let result_value = unsafe { pyre_object::w_int_get_value(boxed_result) }; + + let callable_op = r_args[0]; + if !callable_op.is_constant() { + let expected = ctx.trace_ctx.const_ref(concrete_callable as i64); + ctx.trace_ctx + .record_guard(OpCode::GuardValue, &[callable_op, expected], 0); + walker_capture_snapshot_for_last_guard(ctx, op.pc)?; + ctx.trace_ctx + .heap_cache_mut() + .replace_box(callable_op, expected); + } + let arg_op = r_args[2]; + let float_type_addr = &pyre_object::pyobject::FLOAT_TYPE as *const _ as i64; + let raw_float = walker_unbox_float(ctx, op.pc, arg_op, float_type_addr)?; + walker_guard_exact_w_class( + ctx, + op.pc, + arg_op, + pyre_object::pyobject::get_instantiate(&pyre_object::pyobject::FLOAT_TYPE), + )?; + ctx.trace_ctx + .set_opref_concrete(raw_float, majit_ir::Value::Float(value)); + let low = ctx + .trace_ctx + .const_float(SIGNED_MIN_AS_FLOAT.to_bits() as i64); + let high = ctx + .trace_ctx + .const_float(SIGNED_LIMIT_AS_FLOAT.to_bits() as i64); + walker_float_cmp_guard(ctx, op.pc, OpCode::FloatGe, &[raw_float, low], true)?; + walker_float_cmp_guard(ctx, op.pc, OpCode::FloatLt, &[raw_float, high], true)?; + + let raw_int = ctx + .trace_ctx + .record_op(OpCode::CastFloatToInt, &[raw_float]); + ctx.trace_ctx + .set_opref_concrete(raw_int, majit_ir::Value::Int(result_value)); + let boxed = walker_box_int(ctx, op.pc, raw_int, result_value)?; + ctx.trace_ctx + .set_opref_concrete(boxed, box_int_concrete(result_value, boxed_result as i64)); + write_residual_call_result_to_dst(ctx, op.pc, dst, 'r', boxed)?; + Ok(Some(())) +} + /// `float(x)` on an exact int/float argument: inline the conversion /// (`W_IntObject.descr_float` → `space.newfloat`, or the identity /// `float(f) is f` for an exact float) instead of the opaque diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs index ec9c56c9d51..5aa79551489 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/tests.rs @@ -62,6 +62,89 @@ fn fresh_trace_ctx() -> TraceCtx { TraceCtx::for_test_types(&[Type::Ref]) } +#[test] +fn builtin_wrapper_heapcache_uses_item_not_length_descr() { + let wrapper = named_jitcode("__pyre_wrap_random").expect("random builtin wrapper jitcode"); + let first = crate::jitcode_runtime::decoded_ops(&wrapper.code) + .next() + .expect("wrapper first op"); + assert_eq!(first.key, "arraylen_gc/rd>i"); + let len_pool_index = + wrapper.code[first.pc + 2] as usize | ((wrapper.code[first.pc + 3] as usize) << 8); + let len_descr_index = crate::jitcode_runtime::all_descr_refs()[len_pool_index].index(); + + let item_descr_index = + wrapper_args_item_descr_index(&wrapper.code).expect("wrapper item descriptor"); + assert_ne!( + item_descr_index, len_descr_index, + "Charon slice length and element descriptors are distinct cache keys" + ); + + let getitem = crate::jitcode_runtime::decoded_ops(&wrapper.code) + .find(|op| { + op.key == "getarrayitem_gc_r/rid>r" && wrapper.code.get(op.pc + 1).copied() == Some(0) + }) + .expect("wrapper getarrayitem(r0)"); + let item_pool_index = + wrapper.code[getitem.pc + 3] as usize | ((wrapper.code[getitem.pc + 4] as usize) << 8); + assert_eq!( + item_descr_index, + crate::jitcode_runtime::all_descr_refs()[item_pool_index].index() + ); +} + +#[test] +fn random_core_residuals_use_registered_genrand32_address() { + let expected = pyre_interpreter::jit_trace_fnaddrs() + .into_iter() + .find_map(|(path, address)| { + (path == "module::_random::Random::genrand32").then_some(address) + }) + .expect("genrand32 runtime fnaddr"); + let random = crate::jitcode_runtime::all_jitcodes() + .iter() + .find(|jitcode| { + jitcode.name == "random" + && crate::jitcode_runtime::decoded_ops(&jitcode.code) + .filter(|op| op.key == "residual_call_r_i/iRd>i") + .count() + == 2 + }) + .expect("rrandom Random::random jitcode"); + + assert_eq!( + random + .constants_i + .iter() + .filter(|&&address| address == expected) + .count(), + 1, + "the two genrand32 calls share one runtime-patched constant-pool address" + ); +} + +#[test] +fn user_binop_forward_dunder_covers_fraction_arithmetic_without_inplace_shortcuts() { + use pyre_interpreter::bytecode::BinaryOperator; + + assert_eq!( + user_binop_forward_dunder(BinaryOperator::Subtract), + Some("__sub__") + ); + assert_eq!( + user_binop_forward_dunder(BinaryOperator::TrueDivide), + Some("__truediv__") + ); + assert_eq!( + user_binop_forward_dunder(BinaryOperator::InplaceSubtract), + None + ); + assert_eq!( + user_binop_forward_dunder(BinaryOperator::InplaceTrueDivide), + None + ); +} + /// Build a `done_with_this_frame_descr_ref` for tests. Mirrors the /// production fallback at `pyjitpl.rs` (`make_fail_descr_typed`) /// when the staticdata singleton was never attached. diff --git a/pyre/pyre-jit-trace/src/jitcode_runtime.rs b/pyre/pyre-jit-trace/src/jitcode_runtime.rs index dbfb4896b1e..5174d867128 100644 --- a/pyre/pyre-jit-trace/src/jitcode_runtime.rs +++ b/pyre/pyre-jit-trace/src/jitcode_runtime.rs @@ -120,6 +120,34 @@ pub fn get_jitcode_by_index(index: usize) -> Option> { all_jitcodes().get(index).cloned() } +/// Restore the source translator's exact +/// `Assembler.indirectcalltargets` set as references into `all_jitcodes`. +pub fn build_indirectcalltargets() -> Vec> { + const BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/indirectcalltargets.bin")); + let indices: Vec = bincode::deserialize(BYTES).unwrap_or_else(|e| { + panic!( + "pyre-jit-trace: failed to deserialize indirectcalltargets.bin \ + ({} bytes): {e}", + BYTES.len(), + ) + }); + indices + .into_iter() + .map(|index| { + let canonical = get_jitcode_by_index(index).unwrap_or_else(|| { + panic!( + "pyre-jit-trace: indirect-call target index {index} is \ + outside all_jitcodes (len={})", + all_jitcodes().len() + ) + }); + Arc::new(majit_metainterp::jitcode::JitCode::from_canonical( + (*canonical).clone(), + )) + }) + .collect() +} + // Cached index of the build-time portal jitcode within `ALL_JITCODES`. // // RPython `warmspot.py:281-282` + `call.py:147-148`: diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 4a279d499e2..a47d8b5eab3 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -545,7 +545,20 @@ pub fn blackhole_control_opcodes() -> (i32, i32, i32) { /// `Assembler::indirectcalltargets_vec` in `pyre-jit`. pub fn setup_indirectcalltargets(targets: Vec>) { ensure_finish_setup(); - METAINTERP_SD.with(|r| r.borrow_mut().setup_indirectcalltargets(targets)); + // The source translator and runtime per-CodeObject writer are two Rust + // assembler objects standing in for one RPython CodeWriter assembler. + // Keep the frozen source-translation PBC family when runtime targets are + // published instead of replacing it with the latest runtime-only batch. + let mut merged = crate::jitcode_runtime::build_indirectcalltargets(); + for target in targets { + if !merged + .iter() + .any(|existing| existing.fnaddr == target.fnaddr) + { + merged.push(target); + } + } + METAINTERP_SD.with(|r| r.borrow_mut().setup_indirectcalltargets(merged)); } /// pyjitpl.py:2326-2343 module-level entry point for @@ -6506,6 +6519,59 @@ fn rd_virtual_at( rd_virtuals.and_then(|v| v.get(vidx)).map(|rc| &**rc) } +/// Rebuild the semantic Ref slots of a forced inline frame from its own +/// `locals_cells_stack_w` array. +/// +/// The returned OpRefs are heap reads rooted at `frame_box`, not constants +/// made from the current concrete frame. A compiled bridge therefore reloads +/// the frame image on every guard hit (resume.py:1042-1057 consume_boxes). +fn reconstruct_materialized_frame_slots( + ctx: &mut majit_metainterp::TraceCtx, + frame_box: OpRef, + frame_ref: majit_ir::GcRef, + valuestackdepth: usize, + pending_result_abs_slot: Option, +) -> Option<(Vec, Vec)> { + if frame_ref.is_null() || frame_ref == majit_ir::GcRef::NO_CONCRETE { + return None; + } + let concrete_frame = + unsafe { &*(frame_ref.as_usize() as *const pyre_interpreter::pyframe::PyFrame) }; + let arr_ptr = concrete_frame.locals_cells_stack_w; + if arr_ptr.is_null() { + return None; + } + let arr = unsafe { &*arr_ptr }; + if valuestackdepth > arr.len() { + return None; + } + + let array_box = frame_locals_cells_stack_array(ctx, frame_box); + // `frame_locals_cells_stack_array` currently emits GetfieldRawI for + // backend compatibility, but the field is a GC array Ref + // (virtualizable.py:94). Preserve that boxed-pointer view for the + // following GETARRAYITEM_GC_R operations and executor-side loads. + ctx.try_set_opref_concrete( + array_box, + majit_ir::Value::Ref(majit_ir::GcRef(arr_ptr as usize)), + ); + + let mut registers_r = vec![OpRef::NONE; valuestackdepth]; + let mut concrete_r = vec![majit_ir::Value::Void; valuestackdepth]; + for k in 0..valuestackdepth { + if Some(k) == pending_result_abs_slot { + continue; + } + let index = ctx.const_int(k as i64); + let value = trace_array_getitem_value(ctx, array_box, index); + registers_r[k] = value; + concrete_r[k] = ctx + .box_value(value) + .unwrap_or_else(|| majit_ir::Value::Ref(majit_ir::GcRef(arr.as_slice()[k] as usize))); + } + Some((registers_r, concrete_r)) +} + /// Decode one suspended inline-callee frame's resume section into a /// [`ReconstructRecipe`], or `None` to decline the multi-frame inline rebuild. /// @@ -6697,6 +6763,57 @@ fn reconstruct_inline_recipe( } use majit_ir::resumedata::{RebuiltValue, TAGVIRTUAL, UNINITIALIZED_TAG, untag}; let frame_pos = reg_indices.ref_.iter().position(|&c| c == pframe_reg)?; + // resume.py:1042-1057 rebuild_from_resumedata consumes the saved boxes + // for every frame without requiring the frame object itself to remain + // virtual. A Pyre callee frame can be forced before the guard (for + // example Random.shuffle's loop frame), in which case the frame red is + // a Box failarg rather than TAGVIRTUAL. Its semantic locals/stack are + // still recoverable from that frame's own locals_cells_stack_w array. + // + // Load them symbolically from the failarg-owned frame instead of + // baking the first guard hit's concrete values as constants. The + // reconstructed bridge therefore reads the current frame image on + // every execution, exactly as consume_boxes rebuilds a fresh MIFrame + // from the current deadframe. + if !matches!(values[frame_pos], RebuiltValue::Virtual(_)) { + let (frame_box, frame_value) = bridge_decode_box( + ctx, + values[frame_pos], + Type::Ref, + rd_virtuals, + resume_data, + fail_values, + fail_types, + backend, + cache, + ); + let majit_ir::Value::Ref(frame_ref) = frame_value else { + return None; + }; + ctx.try_set_opref_concrete(frame_box, frame_value); + + let valuestackdepth = nlocals + stack_only; + let (registers_r, concrete_r) = reconstruct_materialized_frame_slots( + ctx, + frame_box, + frame_ref, + valuestackdepth, + pending_result_abs_slot, + )?; + crate::jitcode_dispatch::census_record("P2Recipe::MaterializedFrameAdmit"); + return Some(ReconstructRecipe { + code_ptr: raw_code as *const (), + jitcode_index: frame.jitcode_index, + jitcode_pc: frame.pc, + nlocals, + valuestackdepth, + registers_i: Vec::new(), + registers_r, + registers_f: Vec::new(), + concrete_r, + nargs: nlocals, + }); + } let RebuiltValue::Virtual(frame_vidx) = values[frame_pos] else { return None; }; @@ -10276,6 +10393,59 @@ mod tests { }); } + #[test] + fn materialized_inline_frame_slots_are_symbolic_heap_reads() { + let code = compile_function_body("def f(a, b, c):\n return a if b is None else c\n"); + let mut frame = pyre_interpreter::pyframe::PyFrame::new(code); + let values = [w_int_new(11), pyre_object::w_none(), w_int_new(33)]; + for (index, value) in values.iter().copied().enumerate() { + frame.locals_w_mut()[index] = value; + } + frame.fix_array_ptrs(); + let frame_ptr = (&mut *frame) as *mut pyre_interpreter::pyframe::PyFrame as usize; + + let mut ctx = TraceCtx::for_test_types(&[Type::Ref]); + let frame_box = OpRef::input_arg_ref(0); + ctx.try_set_opref_concrete(frame_box, Value::Ref(majit_ir::GcRef(frame_ptr))); + + let (registers_r, concrete_r) = reconstruct_materialized_frame_slots( + &mut ctx, + frame_box, + majit_ir::GcRef(frame_ptr), + values.len(), + Some(1), + ) + .expect("forced frame slots should be reconstructable"); + + assert!(!registers_r[0].is_none()); + assert!(registers_r[1].is_none()); + assert!(!registers_r[2].is_none()); + assert!( + !matches!(registers_r[0], OpRef::ConstPtr(_)) + && !matches!(registers_r[2], OpRef::ConstPtr(_)), + "bridge slots must reload from the current frame, not bake constants" + ); + assert_eq!( + concrete_r[0], + Value::Ref(majit_ir::GcRef(values[0] as usize)) + ); + assert_eq!(concrete_r[1], Value::Void); + assert_eq!( + concrete_r[2], + Value::Ref(majit_ir::GcRef(values[2] as usize)) + ); + + let tree_loop = ctx.into_tree_loop(); + assert_eq!( + tree_loop + .ops + .iter() + .filter(|op| op.opcode == OpCode::GetarrayitemGcR) + .count(), + 2 + ); + } + #[test] fn concrete_value_preserves_int_subclass_identity() { pyre_interpreter::typedef::init_typeobjects(); @@ -12539,6 +12709,8 @@ pub(crate) fn setup_reconstructed_callee_frame( let frame_vable = crate::helpers::emit_new_pyframe_inline_with_params( ctx, &locals_boxes, + &[], + 0, frame_array_size, nlocals, pycode_const, diff --git a/pyre/pyre-jit-trace/src/trace_opcode.rs b/pyre/pyre-jit-trace/src/trace_opcode.rs index e727d5553d8..e3dfd60d082 100644 --- a/pyre/pyre-jit-trace/src/trace_opcode.rs +++ b/pyre/pyre-jit-trace/src/trace_opcode.rs @@ -2063,7 +2063,9 @@ impl MIFrame { // load a process-global raw word through a baked constant address, // compare it, then guard the result. The folded eval-breaker word is a // bitmask, so this uses a nonzero test rather than upstream's - // `int_lt(ticker, 0)`. + // `int_lt(ticker, 0)`. EB_GC_INTERP is a process-stable dispatch + // configuration bit sharing the word to avoid another interpreter + // load; mask it out because it is not a compiled-loop breaker. // // Load-bearing invariant: RawLoadI must remain outside the always-pure // range and this descriptor must remain non-pure. Otherwise CSE can @@ -2083,7 +2085,9 @@ impl MIFrame { &[base, offset], eval_breaker_word_descr(), ); - let armed = ctx.record_op(OpCode::IntIsTrue, &[word]); + let mask = ctx.const_int(majit_ir::eval_breaker_word::JIT_BREAKER_MASK as i64); + let breaker_bits = ctx.record_op(OpCode::IntAnd, &[word, mask]); + let armed = ctx.record_op(OpCode::IntIsTrue, &[breaker_bits]); self.generate_guard(ctx, OpCode::GuardFalse, &[armed]); } // pyjitpl.py:2954-2965 reached_loop_header: virtualizable_boxes diff --git a/pyre/pyre-jit-trace/src/unpack_state.rs b/pyre/pyre-jit-trace/src/unpack_state.rs index 53cea231d5a..942c9809818 100644 --- a/pyre/pyre-jit-trace/src/unpack_state.rs +++ b/pyre/pyre-jit-trace/src/unpack_state.rs @@ -180,8 +180,10 @@ mod tests { /// unit test. #[test] fn jd1_build_time_descrs_resolve_through_global_pool() { - let canonical = crate::jitcode_runtime::get_jitcode_by_index(0) - .expect("jd1's extracted main JitCode must occupy index 0"); + let canonical = crate::jitcode_runtime::portal_jitcode_for_key( + "baseobjspace::_unpackiterable_unknown_length", + ) + .expect("jd1's extracted main JitCode must be registered"); // The extracted body is the walkable unpack loop: exactly one merge // point, whose byte offset depends on the drain body's op layout // (`UnpackSym::loop_header_pc` discovers it rather than hardcoding). diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 69915c0a0ae..7649d5700cc 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -5073,8 +5073,14 @@ fn set_jit_param_string_via_warmstate(text: &str) -> Result<(), ()> { /// Gate for jd1 (`unpackiterable_driver`): the merge-point hook drives a /// `JitCodeMachine` trace of `_unpackiterable_unknown_length` on hot unpack -/// sites, closing and compiling the drain loop. ON by default, alongside the -/// main JIT. Opt out with `PYRE_NO_JD1` (or `PYRE_JD1=0`); it also follows the +/// sites, closing and compiling the drain loop. This remains opt-in with +/// `PYRE_JD1=1`: unlike RPython, pyre currently drives jd1 through the same +/// `MetaInterp.tracing` slot as the bytecode portal. A residual `next()` on a +/// generator can run an arbitrarily large Python computation before yielding; +/// while that happens the jd1 trace consists only of the opaque `next()` call, +/// but the shared tracing flag suppresses every jd0 merge point reached by the +/// generator body. Keep the incomplete second-driver experiment dormant until +/// it has RPython's independent recursive-portal behavior. It also follows the /// master JIT off-switches (`PYRE_NO_JIT`, `PYRE_JIT=0`) so "no JIT" means no /// jd1. /// @@ -5098,7 +5104,7 @@ fn jd1_experiment_enabled() -> bool { { return false; } - true + std::env::var("PYRE_JD1").as_deref() == Ok("1") }) } @@ -5643,6 +5649,7 @@ pub fn init_jit_hooks() { ); } +#[repr(u8)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum UnsupportedJitShape { None, @@ -5660,6 +5667,27 @@ enum UnsupportedJitShape { ConstEncodingOverflow, } +/// Return the immutable frame-shape classification for an immortal user-code +/// graph. RPython decides the analogous graph facts once while populating +/// `CallControl.jitcodes`; pyre's temporary runtime gate must have the same +/// computed-once lifetime rather than scanning on every Python call. +fn cached_unsupported_jit_shape(code: &pyre_interpreter::CodeObject) -> UnsupportedJitShape { + let key = code as *const _ as usize; + let callcontrol = crate::jit::codewriter::CodeWriter::instance().callcontrol(); + if let Some(&raw) = callcontrol.graph_jit_shapes.get(&key) { + return match raw { + 0 => UnsupportedJitShape::None, + 1 => UnsupportedJitShape::CurrentFrameOnly, + 2 => UnsupportedJitShape::NestedBreakBridgeResume, + 3 => UnsupportedJitShape::ConstEncodingOverflow, + _ => unreachable!("invalid cached UnsupportedJitShape discriminant"), + }; + } + let shape = unsupported_jit_shape(code); + callcontrol.graph_jit_shapes.insert(key, shape as u8); + shape +} + /// True for opcodes that may appear in a `FOR_ITER` loop body without ever /// reaching the orthodox-sub-walk `list.append`/`STORE_SUBSCR` path whose /// walk-abort silently drops an iteration (#57). This is an ALLOW-LIST: @@ -6036,6 +6064,26 @@ fn nested_break_bridge_resume_hazard(code: &pyre_interpreter::CodeObject) -> boo let Some(inner_header) = inner_header else { continue; }; + let Some((pyre_interpreter::Instruction::ForIter { delta }, op_arg)) = + pyre_interpreter::decode_instruction_at(code, inner_header) + else { + continue; + }; + let inner_exit = pyre_interpreter::jump_target_forward( + &code.instructions, + inner_header + 1, + delta.get(op_arg).as_usize(), + ); + // A real inner-loop break pops the inner iterator while control is + // still inside that FOR_ITER's body. If the POP_TOP lies at or beyond + // the inner loop's exhaustion target, the nested loop has already + // ended; a later statement-result POP_TOP followed by the enclosing + // loop's backedge is not a break. This is the shape in testDist: + // an inlined list comprehension ends, then assertTrue() pops its + // result and continues the surrounding product loop. + if pop_pc >= inner_exit { + continue; + } for guard_pc in (inner_header + 1)..pop_pc { match pyre_interpreter::decode_instruction_at(code, guard_pc) { Some((pyre_interpreter::Instruction::PopJumpIfTrue { .. }, _)) => { @@ -6287,7 +6335,8 @@ fn eval_with_jit_inner(frame: &mut PyFrame) -> PyResult { majit_backend_cranelift::register_recovery_layout( crate::call_jit::cranelift_recovery_layout_for_descr, ); - match unsupported_jit_shape(code) { + let jit_shape = cached_unsupported_jit_shape(code); + match jit_shape { UnsupportedJitShape::None => {} UnsupportedJitShape::CurrentFrameOnly => { // Run frames with unsupported current-frame bytecode shapes in the @@ -6622,6 +6671,11 @@ fn eval_loop_jit(frame: &mut PyFrame) -> LoopResult { // handler, etc., where the outer opcode handler holds a PyObjectRef // on the Rust stack that walk_pyframe_roots cannot reach). let _eval_activation = pyre_object::gc_interp::EvalActivationGuard::enter(); + if _eval_activation.armed() { + // Share the interpreter-path GC configuration through the dispatch + // breaker load; the compiled back-edge mask deliberately excludes it. + majit_ir::eval_breaker_word::set_gc_interp(); + } let code = unsafe { &*pyre_interpreter::pyframe_get_pycode(frame_root.frame()) }; // `semantic_loop_headers` is consumed only on the `CloseLoop` arm below (a // back-edge event). Loopless frames — the overwhelming majority during @@ -6636,19 +6690,30 @@ fn eval_loop_jit(frame: &mut PyFrame) -> LoopResult { // No explicit promote needed; the JitDriver green-key mechanism handles this. loop { - pyre_interpreter::module::thread::park_if_finalizing(); + // PyPy's ActionFlag is one process breaker. Keep pyre's free-threaded + // finalization and STW extensions on the same already-established + // breaker word, so the ordinary dispatch pays one relaxed load rather + // than polling two process-global atomics independently. + let dispatch_breaker = majit_ir::eval_breaker_word::load(); + if dispatch_breaker & majit_ir::eval_breaker_word::EB_FINALIZING != 0 { + pyre_interpreter::module::thread::park_if_finalizing(); + } // Interpreter-path GC safepoint (PYRE_GC_INTERP). Between opcodes the // only live refs are in the frame, reachable through the registered // pyframe root walker; no bytecode handler holds a Rust-stack temporary // here. A no-op unless the flag is on and enough interpreter objects // have accumulated to warrant a collection. - pyre_object::gc_interp::safepoint(); + if dispatch_breaker & majit_ir::eval_breaker_word::EB_GC_INTERP != 0 { + pyre_object::gc_interp::safepoint(); + } // Stop-the-world safepoint: a compiled loop's back-edge poll deopts // here when a collector has requested STW; park until it completes. // Between opcodes no bytecode handler holds a Rust-stack ref (see the // note above), so this is a walkable safepoint. - majit_gc::gc_sync::safepoint_poll(); + if dispatch_breaker & majit_ir::eval_breaker_word::EB_STW != 0 { + majit_gc::gc_sync::safepoint_poll(); + } // Seed the frame pointer once after the two top-of-loop safepoints. // The frame is GC-managed and can move only at a collection point; this @@ -7030,10 +7095,13 @@ fn maybe_compile_and_run( if *NO_JIT.get_or_init(|| std::env::var_os("PYRE_NO_JIT").is_some()) { return None; } - let code = unsafe { &*pyre_interpreter::pyframe_get_pycode(frame) }; - if unsupported_jit_shape(code) != UnsupportedJitShape::None { - return None; - } + // `eval_with_jit_inner` classifies the frame once, before entering + // `eval_loop_jit`. Consequently every back-edge reaching this helper is + // already known traceable. Do not repeat `unsupported_jit_shape` here: + // that pyre-only safety gate walks the constant tree and the complete + // bytecode, while RPython's `can_enter_jit` is unconditional. Re-running + // it at every back-edge made the classification scan itself one of + // test_math's hottest native functions. if let Some(expected_vsd) = pyre_jit_trace::state::depth_based_vsd_for_wcode(frame.pycode as usize, loop_header_pc) { @@ -8114,6 +8182,16 @@ fn bound_reached( /// /// Called at every portal entry (function call). Must be fast for the /// common case (no compiled code, not tracing, threshold not reached). +#[inline] +fn dump_bytecode_enabled() -> bool { + // Debug configuration is process-startup state. Reading getenv at every + // Python function entry showed up in test_math's call-heavy Fraction and + // unittest paths; RPython's warmstate entry has no corresponding per-call + // environment lookup. + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var_os("MAJIT_DUMP_BYTECODE").is_some()) +} + pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { let mut frame_root = FrameRoot::new(frame); // warmstate.py parity: PYRE_NO_JIT disables ALL JIT paths. @@ -8122,10 +8200,14 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { return None; } let code = unsafe { &*pyre_interpreter::pyframe_get_pycode(frame_root.frame()) }; - if unsupported_jit_shape(code) != UnsupportedJitShape::None { - return None; - } - if std::env::var_os("MAJIT_DUMP_BYTECODE").is_some() { + // The ordinary caller is `eval_with_jit_inner`, immediately after its + // one authoritative `unsupported_jit_shape` check. The other caller, + // `portal_runner_dispatch`, is recursive re-entry for a portal that could + // only have obtained compiled code after passing that same check. Keep + // this warmstate entry shaped like RPython's unconditional + // `maybe_compile_and_run`; repeating the whole-frame scan here charged + // every Python call even before a warm counter could take the fast path. + if dump_bytecode_enabled() { if code.obj_name.as_str() == "fannkuch" && frame_root.frame().next_instr() == 0 { use std::sync::OnceLock; static DUMPED: OnceLock<()> = OnceLock::new(); @@ -11176,6 +11258,30 @@ mod tests { assert_eq!(unsupported_jit_shape(&code), UnsupportedJitShape::None); } + #[test] + fn nested_break_hazard_ignores_completed_comprehension_before_outer_backedge() { + use pyre_interpreter::compile_exec; + let module = compile_exec( + "def f(rows, pred, sink):\n for p in rows:\n for q in rows:\n diffs = [x for x in q]\n if pred(diffs):\n sink(diffs)\n elif pred(q):\n sink(q)\n", + ) + .expect("test code should compile"); + let code = function_code_from_module(&module, "f"); + + assert!(!nested_break_bridge_resume_hazard(&code)); + } + + #[test] + fn nested_break_hazard_keeps_secondary_edge_inner_break() { + use pyre_interpreter::compile_exec; + let module = compile_exec( + "def f(n):\n total = 0\n for i in range(n):\n for j in range(2, 4):\n if not i % 3 != 0:\n break\n total += j\n return total\n", + ) + .expect("test code should compile"); + let code = function_code_from_module(&module, "f"); + + assert!(nested_break_bridge_resume_hazard(&code)); + } + #[test] fn for_iter_straight_line_store_subscr_body_is_jit_safe() { // The direct STORE_SUBSCR is admitted; a later mid-body abort resumes @@ -12568,7 +12674,7 @@ r = acc", }), "expected an instruction with an ExtendedArg prefix" ); - if std::env::var_os("MAJIT_DUMP_BYTECODE").is_some() { + if dump_bytecode_enabled() { let mut state = pyre_interpreter::OpArgState::default(); for (pc, unit) in code.instructions.iter().copied().enumerate() { let (instr, oparg) = state.get(unit); @@ -12593,7 +12699,7 @@ r = acc", } let mut frame = PyFrame::new(code); let result = eval_with_jit(&mut frame); - if std::env::var_os("MAJIT_DUMP_BYTECODE").is_some() { + if dump_bytecode_enabled() { let mut keys: Vec = unsafe { pyre_object::w_dict_str_entries(frame.get_w_globals()) } .into_iter() diff --git a/pyre/pyre-jit/src/jit/call.rs b/pyre/pyre-jit/src/jit/call.rs index 12112c44e04..a420c1a17af 100644 --- a/pyre/pyre-jit/src/jit/call.rs +++ b/pyre/pyre-jit/src/jit/call.rs @@ -208,6 +208,20 @@ pub struct CallControl { /// with (call.py:29 `self.jitcodes = {}` is a plain instance dict on /// `CallControl`, filled on miss by call.py:155-172 `get_jitcode`). pub loop_header_pcs: HashMap>>, + + /// Pyre-only per-graph result of `eval::unsupported_jit_shape`. + /// + /// RPython has no runtime frame-shape gate: policy and encodability are + /// decided once while `CallControl.jitcodes` is populated. Until pyre's + /// remaining gate arms are removed, keep their likewise immutable result + /// under the same per-graph owner and raw graph key as `jitcodes`, rather + /// than re-walking a user code object's constants and bytecode on every + /// invocation. This is deliberately a `HashMap` because the corresponding + /// upstream owner is `CallControl.jitcodes`, a graph-keyed Python dict; it + /// is not a per-box optimizer side table. Values are the private + /// `UnsupportedJitShape` discriminants, kept as `u8` to avoid making the + /// codewriter layer depend on the portal evaluator. + pub graph_jit_shapes: HashMap, } impl CallControl { @@ -225,6 +239,7 @@ impl CallControl { unfinished_graphs: Vec::new(), callinfocollection: CallInfoCollection::new(), loop_header_pcs: HashMap::new(), + graph_jit_shapes: HashMap::new(), } } diff --git a/pyre/pyre-macros/src/lib.rs b/pyre/pyre-macros/src/lib.rs index 401b028c0e3..a5b4f957766 100644 --- a/pyre/pyre-macros/src/lib.rs +++ b/pyre/pyre-macros/src/lib.rs @@ -1716,6 +1716,7 @@ fn expand_pyre_methods( } let mname = &m.sig.ident; let wrapper_name = format_ident!("__pyre_wrap_{}", mname); + let wrapper_target_name = format_ident!("__majit_builtin_wrapper_target_{}", mname); let kind = classify_method(m)?; // Build per-kind wrapper preamble (self extraction) + call form, @@ -1859,6 +1860,25 @@ fn expand_pyre_methods( ); if has_varargs || is_property { quote! {} + } else if param_names.is_empty() { + let receiver_slots = usize::from(matches!(kind, MethodKind::Instance)); + let expected_total = receiver_slots; + let fn_name = mname.to_string(); + quote! { + // A method with no user-visible parameters has no keyword + // slots to bind. Its valid-call fast path is the exact + // total arity; only a mismatch enters the cold classifier + // that distinguishes CALL_KW's marker from surplus + // positional arguments. + if args.len() != #expected_total { + return crate::gateway::method_noarg_failure( + args, + #fn_name, + #receiver_slots, + ); + } + let __pyre_positional_count = args.len(); + } } else { let mut all_names: Vec = Vec::new(); let mut all_required: Vec = Vec::new(); @@ -1942,10 +1962,14 @@ fn expand_pyre_methods( let expected_min = receiver_slots + visible_required; let too_few = if visible_required == visible_max { quote! { - format!( - "{}() takes {} ({} given)", - #fn_name, #expected, - args.len().saturating_sub(#receiver_slots), + crate::gateway::method_arity_failure( + #fn_name, + #expected, + if args.len() >= #receiver_slots { + args.len() - #receiver_slots + } else { + 0 + }, ) } } else { @@ -1954,19 +1978,23 @@ fn expand_pyre_methods( fn_name, if visible_required == 1 { "" } else { "s" }, ); - quote! { format!(#text, args.len().saturating_sub(#receiver_slots)) } + quote! { + ::std::result::Result::Err(crate::PyError::type_error(format!( + #text, + args.len().saturating_sub(#receiver_slots), + ))) + } }; quote! { if args.len() < #expected_min { - return ::std::result::Result::Err(crate::PyError::type_error(#too_few)); + return #too_few; } if __pyre_positional_count > #expected_total { - return ::std::result::Result::Err(crate::PyError::type_error(format!( - "{}() takes {} ({} given)", + return crate::gateway::method_arity_failure( #fn_name, #expected, - args.len().saturating_sub(#receiver_slots), - ))); + args.len() - #receiver_slots, + ); } } } @@ -2038,6 +2066,16 @@ fn expand_pyre_methods( #body } }); + wrappers.push(quote! { + #[cfg(not(target_arch = "wasm32"))] + #[::linkme::distributed_slice(crate::gateway::BUILTIN_WRAPPER_DESCRIPTORS)] + #[allow(non_upper_case_globals)] + static #wrapper_target_name: crate::gateway::BuiltinWrapperDescriptor = + crate::gateway::BuiltinWrapperDescriptor { + path: concat!(module_path!(), "::", stringify!(#wrapper_name)), + func: #wrapper_name, + }; + }); let raw_fn = quote! { crate::make_builtin_function(#py_name, #wrapper_name) }; match &kind { MethodKind::Instance => { diff --git a/pyre/pyre-object/src/bufferview.rs b/pyre/pyre-object/src/bufferview.rs index dfc8666f404..60aca72b62a 100644 --- a/pyre/pyre-object/src/bufferview.rs +++ b/pyre/pyre-object/src/bufferview.rs @@ -251,8 +251,9 @@ impl BufferView { // step (`strides[0] *= step`, buffer.py:332). BufferView::Slice { parent, step, .. } => { let mut strides = parent.native_strides(); - if let Some(s0) = strides.first_mut() { - *s0 *= *step; + if !strides.is_empty() { + let stride0 = strides[0] * *step; + strides[0] = stride0; } strides } diff --git a/pyre/pyre-object/src/gc_interp.rs b/pyre/pyre-object/src/gc_interp.rs index 75b1cdd87ae..a5d015f8c79 100644 --- a/pyre/pyre-object/src/gc_interp.rs +++ b/pyre/pyre-object/src/gc_interp.rs @@ -85,6 +85,11 @@ impl EvalActivationGuard { } Self { armed } } + + #[inline] + pub fn armed(&self) -> bool { + self.armed + } } impl Drop for EvalActivationGuard { diff --git a/pyre/pyre-object/src/gc_roots.rs b/pyre/pyre-object/src/gc_roots.rs index 064e92135a1..a896a2739e2 100644 --- a/pyre/pyre-object/src/gc_roots.rs +++ b/pyre/pyre-object/src/gc_roots.rs @@ -166,22 +166,35 @@ pub fn shadow_stack_len() -> usize { } /// Read a single shadow-stack slot by index, panicking if the index -/// is out of bounds. Used by tests and ad-hoc host-side debugging to -/// confirm the slot contents survive across nested brackets — the GC -/// itself uses [`walk_shadow_stack`] for the collection-time visit. +/// is out of bounds. RPython's `pop_roots` reads the root-stack slot +/// directly: a moving collection has already rewritten it in place through +/// [`walk_shadow_stack`] (or [`walk_shadow_stack_area`] for another mutator). +/// The initial [`pin_root`] still normalizes a copied pointer that may have +/// become stale before it was published. /// /// Reads the thread-local `SHADOW_STACK` the tracer cannot type; the JIT /// residualises the read instead of tracing into it (`@dont_look_inside`, /// `rlib/jit.py:139`), the [`shadow_stack_len`] twin. #[majit_macros::dont_look_inside] pub fn shadow_stack_get(index: usize) -> PyObjectRef { + SHADOW_STACK.with(|s| s.borrow()[index]) +} + +/// Copy a contiguous range of rooted values out of the shadow stack. +/// +/// RPython's `pop_roots` reloads all livevars from the root stack as one +/// generated block after the potentially-collecting call. Host-side callers +/// that need several adjacent roots should use this equivalent bulk shape +/// instead of repeatedly entering TLS through [`shadow_stack_get`]. +/// +/// Panics when `base..base + dst.len()` is outside the live root stack, just +/// like indexing the corresponding slots individually. +#[majit_macros::dont_look_inside] +pub fn shadow_stack_copy_range(base: usize, dst: &mut [PyObjectRef]) { SHADOW_STACK.with(|s| { - let mut stack = s.borrow_mut(); - let root = stack[index]; - let current = crate::gc_hook::try_gc_current_object_address(root as *mut u8) as PyObjectRef; - stack[index] = current; - current - }) + let stack = s.borrow(); + dst.copy_from_slice(&stack[base..base + dst.len()]); + }); } /// Visit every pinned root in the shadow stack with mutable access.