jit, interp: inline bound methods and defaults, specialize builtin calls, and fold the per-opcode polls into the eval breaker word

Cargo.toml

@@ -180,6 +180,14 @@ tempfile = "3"
 walkdir = "2"
 insta = "1"
 
+# `cargo run --release` is the benchmark/developer performance surface too.
+# Give it the same cross-crate optimization used by distributable binaries;
+# codegen-units=1 additionally lets LLVM optimize the interpreter's large
+# dispatch/call graph as one unit instead of preserving parallel-CGU barriers.
+[profile.release]
+lto = "thin"
+codegen-units = 1
+
 # The profile that 'dist' will build with
 [profile.dist]
 inherits = "release"

majit/majit-gc/src/collector.rs

@@ -113,12 +113,19 @@ fn get_darwin_sysctl_signed(name: &[u8]) -> i64 {
     }
 }
 
-/// env.py:413-433 `get_L2cache_darwin`. Returns the L2+L3 cache size in
-/// bytes via `sysctl`, or -1 when it cannot be determined.
+/// env.py:413-455 `get_L2cache_darwin`. Returns the performance-cluster
+/// L2 plus the legacy L3 cache size via `sysctl`, or -1 when it cannot be
+/// determined.  Apple documents lower performance-level indices as faster
+/// cores, so `hw.perflevel0.l2cachesize` is the cache relevant to the cores
+/// running the mutator.  Intel Macs do not expose that key and retain the
+/// legacy `hw.l2cachesize` fallback.
 #[cfg(target_os = "macos")]
 fn get_l2cache() -> i64 {
-    let mangled = get_darwin_sysctl_signed(b"hw.l2cachesize\0")
-        + get_darwin_sysctl_signed(b"hw.l3cachesize\0");
+    let mut l2cache = get_darwin_sysctl_signed(b"hw.perflevel0.l2cachesize\0");
+    if l2cache <= 0 {
+        l2cache = get_darwin_sysctl_signed(b"hw.l2cachesize\0");
+    }
+    let mangled = l2cache + get_darwin_sysctl_signed(b"hw.l3cachesize\0");
     if mangled > 0 { mangled } else { -1 }
 }
 

majit/majit-ir/src/eval_breaker_word.rs

@@ -5,6 +5,12 @@
 //!                   OR'd in by the OS signal handler and the action dispatcher.
 //!   bit1 EB_STW   — mirrors `GC_SYNC.stw_requested`; OR'd in by the collector
 //!                   while it drains mutators to safepoints.
+//!   bit2 EB_FINALIZING — mirrors interpreter finalization; once armed,
+//!                        non-owner mutators park before their next opcode.
+//!   bit3 EB_GC_INTERP — process-stable `PYRE_GC_INTERP` dispatch gate.  This
+//!                       is masked out of compiled back-edge polls: it avoids
+//!                       a second per-opcode atomic load in the interpreter,
+//!                       but is not itself a reason to leave machine code.
 //! A compiled loop loads the whole word at the back-edge and deopts to the
 //! interpreter when it is non-zero. The interpreter/warm-up loop and the STW
 //! park gate remain authoritative; this word is only the JIT's deopt trigger.
@@ -24,6 +30,12 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 pub const EB_ASYNC: usize = 1;
 /// bit1 — GC stop-the-world requested (mirrors `GC_SYNC.stw_requested`).
 pub const EB_STW: usize = 2;
+/// bit2 — interpreter finalization has begun (terminal, never cleared).
+pub const EB_FINALIZING: usize = 4;
+/// bit3 — interpreter-path allocation/collection integration is enabled.
+pub const EB_GC_INTERP: usize = 8;
+/// Bits that require a compiled loop to deopt to the interpreter.
+pub const JIT_BREAKER_MASK: usize = EB_ASYNC | EB_STW | EB_FINALIZING;
 
 /// The shared eval-breaker word (see module docs).
 static EVAL_BREAKER_WORD: AtomicUsize = AtomicUsize::new(0);
@@ -75,7 +87,17 @@ pub fn clear_stw() {
     EVAL_BREAKER_WORD.fetch_and(!EB_STW, Ordering::Release);
 }
 
+pub fn set_finalizing() {
+    EVAL_BREAKER_WORD.fetch_or(EB_FINALIZING, Ordering::Release);
+}
+
+pub fn set_gc_interp() {
+    EVAL_BREAKER_WORD.fetch_or(EB_GC_INTERP, Ordering::Release);
+}
+
 /// Every flag must fit in the word the poll actually loads. Checked per target,
 /// so a flag too wide for a 32-bit `usize` fails the wasm32 build rather than
 /// silently reading as unarmed there.
-const _: () = assert!((EB_ASYNC | EB_STW) < (1 << (EVAL_BREAKER_WORD_SIZE * 8 - 1)));
+const _: () = assert!(
+    (EB_ASYNC | EB_STW | EB_FINALIZING | EB_GC_INTERP) < (1 << (EVAL_BREAKER_WORD_SIZE * 8 - 1))
+);

majit/majit-metainterp/src/compile.rs

@@ -434,6 +434,15 @@ pub(crate) fn build_guard_metadata<T: AsRef<majit_ir::Op>>(
                     .map_or(false, |d| d.is_resume_guard() || d.is_resume_guard_copied())
                 {
                     fd.set_fail_index_per_trace(fail_index);
+                    // `ops` is the optimized frontend trace retained by
+                    // `CompiledTrace`.  The backend GC rewriter inserts
+                    // operations before code generation, so the assembler's
+                    // prepared-op index is not a valid index into this slice.
+                    // Re-stamp the canonical descr from the frontend op,
+                    // matching RPython where the ResumeGuardDescr remains
+                    // attached to the live ResOperation rather than carrying
+                    // an index into a separate rewritten array.
+                    fd.set_source_op_index(op_idx);
                 }
             }
         }
@@ -2712,6 +2721,33 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_build_guard_metadata_restamps_frontend_source_op_index() {
+        let inputargs = vec![InputArg::new_int(0)];
+        let value = rooted_inputarg_operand(Type::Int, 0);
+        let prefix = Op::new(OpCode::SameAsI, std::slice::from_ref(&value));
+        let descr = make_fail_descr_with_index(0, 1);
+        let fd = descr.as_fail_descr().unwrap();
+        // The GC-rewritten backend trace can insert operations before a
+        // guard and temporarily stamp its prepared-op index on the shared
+        // descr.  That index is not valid in the retained frontend trace.
+        fd.set_source_op_index(99);
+        let mut guard = Op::with_descr(
+            OpCode::GuardTrue,
+            std::slice::from_ref(&value),
+            descr.clone(),
+        );
+        guard.setfailargs(smallvec::smallvec![value]);
+        guard.set_fail_arg_types(vec![Type::Int]);
+
+        let (_resume_data, exit_layouts) =
+            build_guard_metadata(&inputargs, &[prefix, guard], 0, None);
+        let exit = exit_layouts.get(&0).expect("guard exit layout");
+
+        assert_eq!(exit.source_op_index, Some(1));
+        assert_eq!(fd.source_op_index(), Some(1));
+    }
+
     #[test]
     fn test_patch_new_loop_reemits_ops_through_forwarded_results() {
         let mut vinfo = crate::virtualizable::VirtualizableInfo::new(0);

majit/majit-translate/src/codegen.rs

@@ -172,6 +172,7 @@ mod tests {
             }],
             jitcodes: vec![main_jitcode],
             jitcodes_by_path: indexmap::IndexMap::new(),
+            indirectcalltarget_indices: Vec::new(),
             insns: indexmap::IndexMap::new(),
             descrs: Vec::new(),
             all_liveness: Vec::new(),

majit/majit-translate/src/codewriter/call.rs

@@ -570,7 +570,13 @@ impl GraphStore {
     /// signature stays that of the shared graph, which the aliases resolve
     /// to anyway.
     pub(crate) fn insert(&mut self, path: CallPath, graph: FunctionGraph) {
-        let key = (graph.owner_root.clone(), graph.name.clone());
+        let key = (
+            graph
+                .source_identity
+                .clone()
+                .or_else(|| graph.owner_root.clone()),
+            graph.name.clone(),
+        );
         match self.graphs.get_mut(&key) {
             Some(existing) => {
                 existing.graph.func.merge_from(&graph.func);
@@ -3111,6 +3117,17 @@ impl CallControl {
         for path in &todo {
             self.candidate_graphs.insert(path.clone());
         }
+        // PyPy's portal reaches `BuiltinCode.funcrun`, whose `self.func` PBC
+        // contributes every gateway body to the indirect-call candidate set.
+        // Pyre's opcode walker lowers the equivalent Python CALL directly to
+        // `bh_call_fn`, bypassing that source-level dispatch graph, so seed the
+        // same generated-wrapper PBC family explicitly.  This is the builtin
+        // gateway analogue of call.py:59-64's oopspec helper seeds below.
+        for path in self.builtin_wrapper_indirect_graphs() {
+            if self.candidate_graphs.insert(path.clone()) {
+                todo.push(path);
+            }
+        }
         // call.py:59-64 — seed the BFS with builtin oopspec helpers so
         // `int_abs` / `int_floordiv` / `int_mod` / `ll_math.ll_math_sqrt`
         // are reachable even when the portal does not call them
@@ -3252,6 +3269,9 @@ impl CallControl {
                         // `c_graphs` family, `None` meaning "unknown
                         // family" and classifying the site as residual.
                         OpKind::IndirectCall { graphs, .. } => match graphs {
+                            Some(graphs) if graphs.is_empty() => {
+                                self.builtin_wrapper_indirect_graphs()
+                            }
                             Some(graphs) => graphs.clone(),
                             None => continue,
                         },
@@ -3674,6 +3694,16 @@ impl CallControl {
             let arc = self.get_jitcode(&portal);
             self.jitdrivers_sd[jd_index].mainjitcode = Some(arc);
         }
+        // RPython reaches `BuiltinCode.func` as an indirect SomePBC call
+        // while transforming the portal closure; handling that call invokes
+        // `get_jitcode()` for each candidate graph.  Pyre's opcode walker
+        // emits `bh_call_fn` directly and therefore has no source-level
+        // indirect op to perform the allocation.  Materialise the same PBC
+        // family here so runtime fnaddr dispatch can resolve each generated
+        // gateway body to its JitCode.
+        for wrapper in self.builtin_wrapper_indirect_graphs() {
+            self.get_jitcode(&wrapper);
+        }
     }
 
     /// RPython: `CallControl.enum_pending_graphs()` (call.py:150-153).
@@ -4505,6 +4535,35 @@ impl CallControl {
             .collect()
     }
 
+    /// Candidate PBC family for the generated `BuiltinCode.func`
+    /// function-pointer field.
+    ///
+    /// RPython obtains this list from the annotator's `SomePBC`
+    /// descriptions.  Pyre's generated wrappers publish real fnaddrs through
+    /// `jit_trace_fnaddrs`; pair those addresses with their registered source
+    /// graphs here. Aliases sharing an address name the same wrapper; select
+    /// the most-qualified source identity for the one graph object entered
+    /// into the PBC family.
+    pub fn builtin_wrapper_indirect_graphs(&self) -> Vec<CallPath> {
+        let mut by_address: std::collections::BTreeMap<i64, Vec<CallPath>> =
+            std::collections::BTreeMap::new();
+        for (path, &fnaddr) in &self.function_fnaddrs {
+            let Some(leaf) = path.last_segment() else {
+                continue;
+            };
+            if !leaf.starts_with("__pyre_wrap_") || !self.function_graphs.contains_key(path) {
+                continue;
+            }
+            by_address.entry(fnaddr).or_default().push(path.clone());
+        }
+        let mut result = Vec::new();
+        for mut aliases in by_address.into_values() {
+            aliases.sort_by_key(|path| std::cmp::Reverse(path.segments.len()));
+            result.push(aliases.remove(0));
+        }
+        result
+    }
+
     /// RPython `call.py:259-280` — family-wide validation for indirect_call.
     ///
     /// Rejects a family if any member is marked `_elidable_function_` /

majit/majit-translate/src/codewriter/jtransform.rs

@@ -2826,6 +2826,31 @@ impl<'a> Transformer<'a> {
                 kind: OpKind::ConstRefNull,
             }]);
         }
+        // `rbuiltin.py:412-418 rtype_const_result` /
+        // `translator/rtyper/rbuiltin.rs::rtype_ptr_null`: by the time
+        // jtransform runs, `ptr::null[_mut]()` is a typed null pointer
+        // constant, not a residual host call. Pyre's rtyper currently types an
+        // ephemeral oracle rather than rewriting the surviving model graph,
+        // so apply that literal rewrite here. This is the null half of the
+        // niche `Option<NonNull<T>>` / `Option<&T>` representation emitted by
+        // `front::mir`; leaving it as a call would bake an unregistered
+        // symbolic fnaddr into every generated nullity test.
+        if let CallTarget::FunctionPath { segments } = target
+            && args.is_empty()
+            && matches!(result_ty, ValueType::Ref(_))
+            && matches!(
+                segments.as_slice(),
+                [owner, ptr, leaf]
+                    if matches!(owner.as_str(), "core" | "std")
+                        && ptr == "ptr"
+                        && matches!(leaf.as_str(), "null" | "null_mut")
+            )
+        {
+            return RewriteResult::Replace(vec![SpaceOperation {
+                result: op.result.clone(),
+                kind: OpKind::ConstRefNull,
+            }]);
+        }
         // `rewrite_op_cast_pointer` → `rewrite_op_same_as`
         // (jtransform.py:254-257): the JIT does not distinguish a
         // down-cast pointer from its source, so the
@@ -8671,6 +8696,36 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn ptr_null_builtin_rewrites_to_null_ref_constant() {
+        let config = GraphTransformConfig::default();
+        let mut graph = FunctionGraph::new("ptr_null_constant");
+        let entry = graph.startblock;
+        let result_var = graph
+            .push_op_var(
+                entry,
+                OpKind::Call {
+                    target: CallTarget::function_path(["core", "ptr", "null_mut"]),
+                    args: vec![],
+                    result_ty: ValueType::Ref(None),
+                },
+                true,
+            )
+            .unwrap();
+        FunctionGraph::set_concretetype_of_inline(&result_var, ConcreteType::GcRef);
+        graph.set_return(entry, Some(result_var.clone()));
+
+        let result = transform_graph(&graph, &config);
+        let folded = result
+            .graph
+            .blocks
+            .iter()
+            .flat_map(|block| &block.operations)
+            .find(|op| op.result.as_ref() == Some(&result_var))
+            .expect("null result must survive as a constant definition");
+        assert!(matches!(folded.kind, OpKind::ConstRefNull));
+    }
+
     /// PyPy parity regression guard: the qualified spellings
     /// `Result::Ok`, `Option::Some`, `std::result::Result::Err` etc.
     /// must elide identically to the bare `Ok` / `Some` / `Err`