diff --git a/Cargo.lock b/Cargo.lock index c6c877b6ad1..b54d475b565 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2849,6 +2849,7 @@ dependencies = [ "indexmap", "linkme", "majit-gc", + "majit-ir", "majit-macros", "majit-rlib", "num-traits", diff --git a/majit/majit-backend-cranelift/src/compiler.rs b/majit/majit-backend-cranelift/src/compiler.rs index b4b8a347664..ecab32caddc 100644 --- a/majit/majit-backend-cranelift/src/compiler.rs +++ b/majit/majit-backend-cranelift/src/compiler.rs @@ -16103,19 +16103,23 @@ fn collect_guards( // also contain Const (handled by regalloc.py:1192-1193 in the same // loop). majit groups external JUMP with FINISH for fail_args // bookkeeping; treat their gcmap the same way. - // `compute_gcmap` skips the hole a virtual leaves (`if arg is None: - // continue`), marks every remaining REF failarg, and narrows nothing - // else. No hole reaches this map: `spill_guard_fail_args` resolves - // every fail arg through `resolve_opref`, which refuses `OpRef::NONE` - // rather than substituting a zero, so a guard carrying one fails to - // compile before a gcmap exists. A force token is REF too - // (`FORCE_TOKEN/0/r` returns the jitframe, a moving GC object), so its - // slot is marked like any other. + // `compute_gcmap` opens with `if arg is None: continue`, then marks + // every remaining REF failarg and narrows nothing else. The skip is + // load-bearing here rather than vacuous: a virtual leaves a hole in + // fail_args, and both `infer_fail_arg_types` and + // `resolve_fail_arg_types` type that hole `Ref` (a virtual object is a + // GCREF), so the type test on its own would mark a slot that owns no + // root. A force token is REF too (`FORCE_TOKEN/0/r` returns the + // jitframe, a moving GC object), so its slot is marked like any other. let failarg_ref_slots = { let mut slots = Vec::new(); for (i, tp) in fail_arg_types.iter().enumerate() { if *tp == Type::Ref { let arg_ref = fail_arg_refs.get(i).copied().unwrap_or(OpRef::NONE); + // `assembler.py compute_gcmap` `if arg is None: continue`. + if arg_ref.is_none() { + continue; + } // regalloc.py:1206 — guard fail_args must never be Const. // history.py/268/314 inline-Const carries the value on // the OpRef itself; legacy idx-Const lives in `constants`. diff --git a/majit/majit-backend/src/lib.rs b/majit/majit-backend/src/lib.rs index e1db2b438cf..38b6d4945f4 100644 --- a/majit/majit-backend/src/lib.rs +++ b/majit/majit-backend/src/lib.rs @@ -1648,6 +1648,17 @@ impl std::fmt::Debug for JitCellToken { } } +impl majit_ir::QuasiImmutLoopToken for JitCellToken { + fn invalidate_for_quasi_immut(&self) { + // quasiimmut.py `QuasiImmut.invalidate`: `looptoken.invalidated = True` + // followed by `cpu.invalidate_loop(looptoken)`. `invalidate` performs both + // projections in pyre: the root flag makes the warm cell stop + // returning this token, and every bridge-generation flag activates + // its still-unpatched GUARD_NOT_INVALIDATED sites. + self.invalidate(); + } +} + // pyre is single-threaded (no-GIL → still one JIT thread in practice, // matching RPython's single-interpreter assumption). `JitCellToken` // embeds `Rc` and `Box` which are not diff --git a/majit/majit-gc/src/collector.rs b/majit/majit-gc/src/collector.rs index 2bab68bf5f4..793c67d7c2d 100644 --- a/majit/majit-gc/src/collector.rs +++ b/majit/majit-gc/src/collector.rs @@ -1806,7 +1806,11 @@ impl MiniMarkGC { let type_id = unsafe { (*header_of(pinned_obj)).type_id() }; let payload_size = self.size_for_typeid(pinned_obj, type_id, "pinned_barriers"); let object_size = Self::nursery_allocation_size(GcHeader::SIZE + payload_size); - let next_free = pinned_header + object_size; + // `size_for_typeid` decodes the pinned object's extent from its + // header. A decode that overstates it would push free past the + // barrier we are about to publish, and `Nursery::alloc` would then + // hand out bytes beyond the gap. The barrier is the hard bound. + let next_free = (pinned_header + object_size).min(next_top); unsafe { // Set the wider bound first so Nursery's pointer invariant is // maintained while free crosses the old (pinned) top. @@ -2535,10 +2539,9 @@ impl MiniMarkGC { self.nursery_surviving_size = 0; // `IncrementalMiniMarkGC._minor_collection`: pinning does not keep an // object alive. Rebuild the AddressStack and count from traced edges. - // pyre currently walks the complete root stacks on every minor, so the - // saved stopper decision is conservatively unused; keep the state with - // the collector, where upstream owns it. - let _any_pinned_object_from_earlier = self.any_pinned_object_kept; + // The flag sampled here is the previous minor's, which + // `collect_roots_in_nursery` turns into `use_jit_frame_stoppers`. + let any_pinned_object_from_earlier = self.any_pinned_object_kept; self.surviving_pinned_objects.clear(); self.pinned_objects_in_nursery = 0; self.any_pinned_object_kept = false; @@ -2678,9 +2681,18 @@ impl MiniMarkGC { // a minor collection (incminimark.py:339-344 // `old_objects_pointing_to_young`); restored to the conservative // Major default right after. - crate::shadow_stack::set_extra_root_walk_kind( - crate::shadow_stack::ExtraRootWalkKind::Minor, - ); + // + // `collect_roots_in_nursery` computes + // `use_jit_frame_stoppers = not any_pinned_object_from_earlier` and + // passes it as `is_minor`: a pinned object created before the previous + // minor is still in the nursery and was never promoted, so the skip + // would drop the only edge reaching it. Announce a full walk instead. + let extra_root_walk_kind = if any_pinned_object_from_earlier { + crate::shadow_stack::ExtraRootWalkKind::Major + } else { + crate::shadow_stack::ExtraRootWalkKind::Minor + }; + crate::shadow_stack::set_extra_root_walk_kind(extra_root_walk_kind); let mut visit_extra_area = |gcref: &mut GcRef| { self.drag_out_root(gcref); }; @@ -11707,6 +11719,56 @@ cache size\t: 8192 kB\n"; gc.roots.clear(); } + /// `collect_roots_in_nursery` computes + /// `use_jit_frame_stoppers = not any_pinned_object_from_earlier` and passes + /// it to `walk_roots` as `is_minor`. A pin that survived the previous minor + /// was never promoted and still sits at its nursery address, so the walkers + /// that skip a clean area on a minor have to be told to walk everything. + #[test] + fn a_surviving_pin_makes_the_next_minor_announce_a_full_extra_root_walk() { + // The walker registry has no removal and every test binary thread + // shares it, so record on the collecting thread only. + thread_local! { + static SEEN: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + } + fn record(_visit: &mut dyn FnMut(&mut GcRef)) { + SEEN.with(|seen| { + seen.borrow_mut() + .push(crate::shadow_stack::extra_root_walk_kind()) + }); + } + + let _guard = SHADOW_STACK_TEST_LOCK.lock().unwrap(); + crate::shadow_stack::clear(); + crate::shadow_stack::register_extra_root_walker(record); + SEEN.with(|seen| seen.borrow_mut().clear()); + + let mut gc = test_gc(4096); + let tid = gc.register_type(TypeInfo::simple(16)); + let mut obj = gc.alloc_with_type(tid, 16); + assert!(gc.pin(obj)); + unsafe { gc.roots.add(&mut obj) }; + + // Nothing was pinned before this one, so the first minor still + // announces the skip; it discovers the pin and leaves it in place. + gc.do_collect_nursery(); + assert!(gc.is_pinned(obj)); + + // The pin now predates the previous minor: it is "from earlier". + gc.do_collect_nursery(); + assert!(gc.is_pinned(obj)); + + gc.roots.clear(); + assert_eq!( + SEEN.with(|seen| seen.borrow().clone()), + vec![ + crate::shadow_stack::ExtraRootWalkKind::Minor, + crate::shadow_stack::ExtraRootWalkKind::Major, + ] + ); + } + /// The sibling above only ever discovers the parent *after* the list is /// swapped out. Phase 1c traces an old-generation jitframe directly, with /// itself as the holder, and that runs earlier — so a parent found there diff --git a/majit/majit-gc/src/nursery.rs b/majit/majit-gc/src/nursery.rs index 70f30b463d4..879c417962e 100644 --- a/majit/majit-gc/src/nursery.rs +++ b/majit/majit-gc/src/nursery.rs @@ -149,10 +149,17 @@ impl Nursery { debug_assert!(start >= self.start as usize); debug_assert!(start <= end); debug_assert!(end <= self.start as usize + self.size); - let len = end - start; - if len == 0 { + // This is a safe fn that writes raw bytes at caller-supplied + // addresses, so the bounds have to hold in release too: an + // out-of-range end would write outside the arena, and `start > end` + // would wrap the length into a near-`usize::MAX` fill. Intersect with + // the arena instead of trusting the caller. + let lo = start.max(self.start as usize); + let hi = end.min(self.start as usize + self.size); + if lo >= hi { return; } + let (start, len) = (lo, hi - lo); #[cfg(target_arch = "wasm32")] unsafe { ptr::write_bytes(start as *mut u8, 0, len); diff --git a/majit/majit-ir/src/descr.rs b/majit/majit-ir/src/descr.rs index 07dab5b14e3..bb0879fcebb 100644 --- a/majit/majit-ir/src/descr.rs +++ b/majit/majit-ir/src/descr.rs @@ -3684,6 +3684,20 @@ pub trait Descr: Send + Sync + std::fmt::Debug { } } +/// `history.py JitCellToken` as seen by `quasiimmut.py QuasiImmut`. +/// +/// A quasi-immutable dependency belongs to the owning loop token, never to +/// one machine-code fragment. `QuasiImmut.invalidate` marks that token +/// invalid and asks the CPU to activate every still-unpatched +/// `GUARD_NOT_INVALIDATED` in the loop and its bridges. Keeping this as a +/// trait avoids making the interpreter object model depend on a backend +/// implementation while preserving the upstream ownership shape. +pub trait QuasiImmutLoopToken: Send + Sync + std::fmt::Debug { + /// `quasiimmut.py QuasiImmut.invalidate` — + /// `looptoken.invalidated = True; cpu.invalidate_loop(looptoken)`. + fn invalidate_for_quasi_immut(&self); +} + /// `quasiimmut.py QuasiImmut` seen from the JIT side — one object /// gathering the loops that folded a single quasi-immutable field. /// @@ -3698,7 +3712,7 @@ pub trait QuasiImmutHandle: Send + Sync + std::fmt::Debug { /// `quasiimmut.py register_loop_token`, reached from /// `compile.py:204-207`. - fn register_loop_token(&self, flag: &std::sync::Arc); + fn register_loop_token(&self, token: &std::sync::Arc); /// Identity of the instance behind the handle. /// diff --git a/majit/majit-ir/src/lib.rs b/majit/majit-ir/src/lib.rs index afd769ad2ae..0e3519b86c7 100644 --- a/majit/majit-ir/src/lib.rs +++ b/majit/majit-ir/src/lib.rs @@ -30,13 +30,13 @@ pub mod value; pub use descr::{ AccumInfo, ArrayDescr, ArrayFlag, CallDescr, DebugMergePointDescr, DebugMergePointInfo, Descr, DescrRef, FailDescr, FailDescrCell, FieldDescr, GcCache, InteriorFieldDescr, JitCodeDescr, - LLType, LoopTargetDescr, LoopTokenDescr, QuasiImmutDescr, QuasiImmutHandle, SimpleCallDescr, - SimpleFailDescr, SimpleFieldDescr, SizeDescr, SwitchDescr, TargetArgLoc, UnpackAtExitInfo, - descr_identity, make_array_descr, make_array_descr_signed, make_call_descr, make_field_descr, - make_field_descr_full, make_loop_target_descr, make_malloc_array_calldescr, - make_malloc_array_nonstandard_calldescr, make_malloc_big_fixedsize_calldescr, - make_malloc_str_calldescr, make_malloc_unicode_calldescr, make_memcpy_calldescr, - make_size_descr_full, make_size_descr_with_vtable, make_tid_field_descr, + LLType, LoopTargetDescr, LoopTokenDescr, QuasiImmutDescr, QuasiImmutHandle, + QuasiImmutLoopToken, SimpleCallDescr, SimpleFailDescr, SimpleFieldDescr, SizeDescr, + SwitchDescr, TargetArgLoc, UnpackAtExitInfo, descr_identity, make_array_descr, + make_array_descr_signed, make_call_descr, make_field_descr, make_field_descr_full, + make_loop_target_descr, make_malloc_array_calldescr, make_malloc_array_nonstandard_calldescr, + make_malloc_big_fixedsize_calldescr, make_malloc_str_calldescr, make_malloc_unicode_calldescr, + make_memcpy_calldescr, make_size_descr_full, make_size_descr_with_vtable, make_tid_field_descr, make_vtable_field_descr, memcpy_fn_addr, recover_fail_descr_cell, unpack_fielddescr, }; pub use effectinfo::{ diff --git a/majit/majit-metainterp/src/compile.rs b/majit/majit-metainterp/src/compile.rs index d918c7f5e96..2f950308235 100644 --- a/majit/majit-metainterp/src/compile.rs +++ b/majit/majit-metainterp/src/compile.rs @@ -1658,9 +1658,10 @@ pub(crate) fn infer_terminal_exit_layout>( .iter() .map(|opref| { // `OpRef::NONE` represents a null-ref placeholder per - // `fail_arg_type`; preserve `Type::Ref` so the gcmap and - // `decode_values_with_layout` see the same null-Ref typing the - // rest of the resume path uses. + // `fail_arg_type`; preserve `Type::Ref` so every consumer of this + // layout sees the same null-Ref typing the rest of the resume + // path uses. The gcmap is not one of them: `compute_gcmap` drops + // a `None` failarg before it reads the type at all. if opref.is_none() { return Type::Ref; } @@ -1724,49 +1725,6 @@ pub(crate) fn build_terminal_exit_layouts>( layouts } -#[allow(dead_code)] -pub(crate) fn terminal_exit_layout_for_trace( - trace: &CompiledTrace, - owning_key: u64, - trace_id: u64, - op_index: usize, -) -> Option { - if let Some(layout) = trace.terminal_exit_layouts.get(&op_index) { - return Some(layout.public( - owning_key, - trace_id, - find_fail_index_for_exit_op(&trace.ops, op_index).unwrap_or(u32::MAX), - )); - } - if let Some(fail_index) = find_fail_index_for_exit_op(&trace.ops, op_index) - && let Some(layout) = trace.exit_layouts.get(&fail_index) - { - return Some(layout.public(owning_key, trace_id, fail_index)); - } - infer_terminal_exit_layout(&trace.inputargs, &trace.ops, owning_key, trace_id, op_index) -} - -#[allow(dead_code)] -pub(crate) fn decode_values_with_layout( - raw_values: &[i64], - layout: &CompiledExitLayout, -) -> Vec { - layout - .exit_types - .iter() - .enumerate() - .map(|(index, tp)| { - let raw = raw_values.get(index).copied().unwrap_or(0); - match tp { - Type::Int => Value::Int(raw), - Type::Ref => Value::Ref(GcRef(raw as usize)), - Type::Float => Value::Float(f64::from_bits(raw as u64)), - Type::Void => Value::Void, - } - }) - .collect() -} - pub(crate) fn normalize_closing_jump_args( ops: Vec, constants: &majit_ir::ConstMap, diff --git a/majit/majit-metainterp/src/jitcode/assembler.rs b/majit/majit-metainterp/src/jitcode/assembler.rs index c6c867875ab..3a797dd4bbb 100644 --- a/majit/majit-metainterp/src/jitcode/assembler.rs +++ b/majit/majit-metainterp/src/jitcode/assembler.rs @@ -6550,8 +6550,6 @@ mod tests { /// — the offset is only a stand-in for the mint sites that carry no name. #[test] fn a_named_field_resolves_by_name_through_an_ambiguous_offset() { - #[allow(dead_code)] - const TID: u64 = 0x4E41_4D45_4B59; let fields = [ (0, false, "head", 8, false), (8, false, "agg", 8, true), diff --git a/majit/majit-metainterp/src/jitdriver.rs b/majit/majit-metainterp/src/jitdriver.rs index d4541aac713..f50ce55a9c0 100644 --- a/majit/majit-metainterp/src/jitdriver.rs +++ b/majit/majit-metainterp/src/jitdriver.rs @@ -1388,10 +1388,11 @@ pub struct JitDriver { /// traces from an exception guard (GUARD_EXCEPTION / GUARD_NO_EXCEPTION). /// The caller should emit SAVE_EXC_CLASS + SAVE_EXCEPTION at trace start. pub last_bridge_is_exception_guard: bool, - /// Recorded op count after bridge setup prologue materialization and - /// before the bridge body walk starts. Used to distinguish deterministic - /// setup aborts from transient mid-trace aborts. - bridge_body_start_op_count: Option, + /// `(current, monotonic)` recorded-op counts after bridge setup prologue + /// materialization and before the bridge body walk starts. Used to + /// distinguish deterministic setup aborts from transient mid-trace aborts + /// even when `history.cut` rewinds the current count to the setup position. + bridge_body_start_op_counts: Option<(usize, usize)>, /// Whether this session's bridge attempt was declined, or had no target to /// close against -- the session PHASE, as distinct from the resumekey CLASS /// that `MetaInterp::bridge_info` carries. @@ -1607,7 +1608,7 @@ impl JitDriver { bridge_entered_at_guard_resume: false, resume_data_result: None, last_bridge_is_exception_guard: false, - bridge_body_start_op_count: None, + bridge_body_start_op_counts: None, bridge_attempt_declined: false, entry_points: Vec::new(), is_recursive: false, @@ -4180,12 +4181,14 @@ impl JitDriver { // permanently decline the source guard. && self.meta.partial_trace().is_none() && self.meta.tracing.as_ref().is_some_and(|ctx| { - let Some(start_ops) = self.bridge_body_start_op_count else { + let Some((start_ops, start_total)) = self.bridge_body_start_op_counts else { return false; }; let current_ops = ctx.num_ops(); - current_ops == start_ops + let current_total = ctx.recorded_ops_total(); + (current_ops == start_ops && current_total == start_total) || (current_ops == start_ops + 1 + && current_total == start_total + 1 && ctx .ops() .get(start_ops) @@ -4354,7 +4357,7 @@ impl JitDriver { self.meta.aborted_tracing(reason_int); } self.sym = None; - self.bridge_body_start_op_count = None; + self.bridge_body_start_op_counts = None; self.bridge_attempt_declined = false; self.meta.clear_trace_session(); } @@ -4368,7 +4371,7 @@ impl JitDriver { self.meta.abort_trace(true); self.sym = None; // The session ends here, so the latch must not outlive it. The - // sibling `bridge_body_start_op_count` is cleared on the + // sibling `bridge_body_start_op_counts` is cleared on the // Abort/Decline arm and in `clear_tracing_session_state` but // not here, which is why this arm needs its own reset rather // than a shared teardown. @@ -4403,7 +4406,7 @@ impl JitDriver { #[inline(never)] fn clear_tracing_session_state(&mut self) { self.sym = None; - self.bridge_body_start_op_count = None; + self.bridge_body_start_op_counts = None; self.bridge_attempt_declined = false; self.meta.clear_trace_session(); } @@ -6909,65 +6912,6 @@ impl JitDriver { } } - #[allow(dead_code)] - fn decode_descriptor_values( - descriptor: Option<&JitDriverStaticData>, - raw_values: &[i64], - ) -> Option> { - let descriptor = descriptor?; - let reds = descriptor.reds(); - if reds.len() != raw_values.len() { - return None; - } - Some( - reds.iter() - .zip(raw_values.iter().copied()) - .map(|(var, raw)| match var.tp { - Type::Int => Value::Int(raw), - Type::Ref => Value::Ref(majit_ir::GcRef(raw as usize)), - Type::Float => Value::Float(f64::from_bits(raw as u64)), - Type::Void => Value::Void, - }) - .collect(), - ) - } - - #[allow(dead_code)] - fn decode_exit_layout_values(raw_values: &[i64], layout: &CompiledExitLayout) -> Vec { - layout - .exit_types - .iter() - .enumerate() - .map(|(index, tp)| { - let raw = raw_values.get(index).copied().unwrap_or(0); - match tp { - Type::Int => Value::Int(raw), - Type::Ref => Value::Ref(majit_ir::GcRef(raw as usize)), - Type::Float => Value::Float(f64::from_bits(raw as u64)), - Type::Void => Value::Void, - } - }) - .collect() - } - - #[allow(dead_code)] - fn resume_layout_with_descriptor_slot_types( - descriptor: Option<&JitDriverStaticData>, - resume_layout: &ResumeLayoutSummary, - ) -> Option { - let descriptor = descriptor?; - let red_types: Vec = descriptor.reds().iter().map(|var| var.tp).collect(); - let last = resume_layout.frame_layouts.last()?; - if last.slot_types.is_some() || last.slot_layouts.len() != red_types.len() { - return None; - } - let mut patched = resume_layout.clone(); - if let Some(last) = patched.frame_layouts.last_mut() { - last.slot_types = Some(red_types); - } - Some(patched) - } - /// Invalidate a compiled loop, forcing fallback to interpretation. pub fn invalidate_loop(&mut self, green_key: u64) { self.meta.invalidate_loop(green_key); @@ -7112,15 +7056,15 @@ impl JitDriver { self.meta.last_compiled_key() } - /// Flag read by `GUARD_NOT_INVALIDATED` in the last successful artifact. - pub fn last_compiled_artifact_invalidation_flag( + /// Owning loop token of the last successful loop/bridge artifact. + pub fn last_compiled_artifact_token( &self, - ) -> Option> { - self.meta.last_compiled_artifact_invalidation_flag() + ) -> Option> { + self.meta.last_compiled_artifact_token() } - pub fn clear_last_compiled_artifact_invalidation_flag(&mut self) { - self.meta.clear_last_compiled_artifact_invalidation_flag(); + pub fn clear_last_compiled_artifact_token(&mut self) { + self.meta.clear_last_compiled_artifact_token(); } /// warmstate.py:437-444 starting cell's green_key (the cell on which @@ -7537,7 +7481,7 @@ impl JitDriver { // Same reason as the primary trace entry: the bridge compile decodes // frame value counts through the per-thread store, so aim it here. self.republish_state_field_fvc(); - self.bridge_body_start_op_count = None; + self.bridge_body_start_op_counts = None; self.bridge_attempt_declined = false; // compile.py `_trace_and_compile_from_bridge` raises // `compile.giveup()` when the descr's owning JitCellToken weakref @@ -7924,7 +7868,11 @@ impl JitDriver { &retrace.fail_types, ); } - self.bridge_body_start_op_count = self.meta.tracing.as_ref().map(|ctx| ctx.num_ops()); + self.bridge_body_start_op_counts = self + .meta + .tracing + .as_ref() + .map(|ctx| (ctx.num_ops(), ctx.recorded_ops_total())); self.meta.begin_trace_session(trace_meta); // resume.py:1047-1055 parity: // ResumeDataBoxReader.consume_boxes() rebuilds the frame state, diff --git a/majit/majit-metainterp/src/optimizeopt/guard.rs b/majit/majit-metainterp/src/optimizeopt/guard.rs index f8d10ce320f..e8cd3f3a3e9 100644 --- a/majit/majit-metainterp/src/optimizeopt/guard.rs +++ b/majit/majit-metainterp/src/optimizeopt/guard.rs @@ -540,15 +540,6 @@ impl GuardStrengthenOpt { } } - #[allow(dead_code)] - fn set_guard( - guards: &mut indexmap::IndexMap>, - idx: usize, - val: Option, - ) { - guards.insert(idx, val); - } - /// guard.py: eliminate_guards(loop) pub fn eliminate_guards(&mut self, ops: &[Op]) -> Vec { // guard.py:222: self.renamer = Renamer() diff --git a/majit/majit-metainterp/src/optimizeopt/heap.rs b/majit/majit-metainterp/src/optimizeopt/heap.rs index f7619d78861..8d8875d8e96 100644 --- a/majit/majit-metainterp/src/optimizeopt/heap.rs +++ b/majit/majit-metainterp/src/optimizeopt/heap.rs @@ -4689,8 +4689,7 @@ mod tests { } /// Same shape as `test_getfield_read_after_read`, but through a descr that - /// owns a parent SizeDescr and sits at a non-zero slot — the production - /// shape (`PyFrame.execution_context` is `index_in_parent: 7`). The + /// owns a parent SizeDescr and sits at a non-zero slot. The /// parentless `TestDescr` used by the sibling test takes /// `field_slot_index`'s `Descr::index()` fallback, so it never exercises /// the `index_in_parent` slot that `ensure_ptr_info_arg0` sizes the @@ -4795,10 +4794,9 @@ mod tests { /// it instead of upgrading, so the write is dropped and every later read of /// the same field misses. /// - /// `inline_helper` traces exactly this: six `getfield_gc_r(p0, - /// PyFrame.execution_context)` off one frame, none folded, which keeps the - /// frame push/pop `topframeref` stores from cancelling and forces the - /// virtual `VRef` that `NewWithVtable`/`ForceToken` then materialise. + /// Inline-frame helpers exercise the same general shape for ordinary heap + /// fields read from a virtualizable receiver; those reads must still fold + /// exactly as they do on a plain `InstancePtrInfo` receiver. #[test] fn getfield_off_a_virtualizable_receiver_is_still_cached() { use crate::optimizeopt::info::{PtrInfo, VirtualizableFieldState}; diff --git a/majit/majit-metainterp/src/optimizeopt/unroll.rs b/majit/majit-metainterp/src/optimizeopt/unroll.rs index 4c1342c657a..ab91d93dff7 100644 --- a/majit/majit-metainterp/src/optimizeopt/unroll.rs +++ b/majit/majit-metainterp/src/optimizeopt/unroll.rs @@ -2270,7 +2270,7 @@ mod quasi_immutable_dep_tests { true } - fn register_loop_token(&self, _flag: &std::sync::Arc) {} + fn register_loop_token(&self, _token: &std::sync::Arc) {} fn instance_identity(&self) -> usize { std::sync::Arc::as_ptr(&self.0) as usize @@ -2564,13 +2564,6 @@ impl ExportedState { } } - #[allow(dead_code)] - fn visit_value(value: &mut Value, visitor: &mut dyn FnMut(&mut GcRef)) { - if let Value::Ref(gcref) = value { - visitor(gcref); - } - } - fn visit_op(op: &Op, visitor: &mut dyn FnMut(&mut GcRef)) { let mut pos = op.pos.get(); visit_opref(&mut pos, visitor); diff --git a/majit/majit-metainterp/src/optimizeopt/vstring.rs b/majit/majit-metainterp/src/optimizeopt/vstring.rs index 241b62a9978..9b4e9176686 100644 --- a/majit/majit-metainterp/src/optimizeopt/vstring.rs +++ b/majit/majit-metainterp/src/optimizeopt/vstring.rs @@ -377,11 +377,6 @@ impl OptString { } } - #[allow(dead_code)] - fn is_virtual_concat(&self, op: &Operand, ctx: &OptContext) -> bool { - self.get_concat_info(op, ctx).is_some() - } - fn get_slice_info(&self, op: &Operand, ctx: &OptContext) -> Option { match ctx.peek_ptr_info(op) { Some(PtrInfo::Str(sinfo)) => match sinfo.variant { @@ -392,11 +387,6 @@ impl OptString { } } - #[allow(dead_code)] - fn is_virtual_slice(&self, op: &Operand, ctx: &OptContext) -> bool { - self.get_slice_info(op, ctx).is_some() - } - /// vstring.py: read the string mode (0 = byte string, 1 = unicode) from /// the installed `StrPtrInfo`. Returns 0 when no PtrInfo is set — callers /// inside the pass only hit this path for constant/forwarded refs where diff --git a/majit/majit-metainterp/src/pyjitpl.rs b/majit/majit-metainterp/src/pyjitpl.rs index e00e8abe358..7df33102636 100644 --- a/majit/majit-metainterp/src/pyjitpl.rs +++ b/majit/majit-metainterp/src/pyjitpl.rs @@ -1637,18 +1637,20 @@ pub struct MetaInterp { /// from the tracing green_key when cross-loop cut retargets to the /// inner loop's key (compile.py:269). pub(crate) last_compiled_key: Option, - /// Invalidation flag read by the most recently compiled loop or bridge. - /// Dependency registration uses the artifact generation rather than - /// always registering the root token's flag. - pub(crate) last_compiled_artifact_invalidation_flag: Option>, - /// pyjitpl.py: trace position saved before compile_trace records + /// Owning JitCellToken of the most recently compiled loop or bridge. + /// `compile.py:record_loop_or_bridge` registers this token itself on every + /// quasi-immutable dependency, including when the dependency was found in + /// an attached bridge. Backend invalidation generations are deliberately + /// not exposed here: they describe guard patching, not dependency owner. + pub(crate) last_compiled_artifact_token: Option>, + /// pyjitpl.py `compile_trace` position saved before it records /// a tentative JUMP. If compile_trace triggers retrace_needed, this /// becomes the retracing_from position. pub(crate) potential_retrace_position: Option, /// RPython compile.py (record_loop_or_bridge) parity: /// quasi-immutable dependencies from the last compilation — the /// `QuasiImmut` instances the recording resolved. After compilation, the - /// caller registers the loop's invalidation flag on each. Cleared on each + /// caller registers the owning loop token on each. Cleared on each /// compile attempt. pub last_quasi_immutable_deps: Vec>, /// Addresses of live `SnapshotBox.opref` slots holding an inline @@ -3194,7 +3196,7 @@ impl MetaInterp { cancel_count: 0, internal_compile_panics: 0, last_compiled_key: None, - last_compiled_artifact_invalidation_flag: None, + last_compiled_artifact_token: None, potential_retrace_position: None, last_quasi_immutable_deps: Vec::new(), compile_snapshot_refs: Vec::new(), @@ -4647,7 +4649,7 @@ impl MetaInterp { fn prepare_trace_start_runtime(&mut self) { self.last_compiled_key = None; - self.last_compiled_artifact_invalidation_flag = None; + self.last_compiled_artifact_token = None; // pyjitpl.py `compile_and_run_once` body, line-by-line: // debug_start('jit-tracing') # OUTER open // self.staticdata._setup_once() @@ -4719,25 +4721,19 @@ impl MetaInterp { match hot { HotResult::NotHot => BackEdgeAction::Interpret, HotResult::StartTracing => { - self.prepare_trace_start_runtime(); - // `force_start_tracing_for_key` has just installed (or found) - // this key's cell, so resolving now names it. The number that - // arrived is a bucket hash, and a bucket holds a chain: if any - // other cell already claimed that hash, the cell just marked - // TRACING was minted a different key. The trace, its - // `compiled_loops` entry, its `JitCellToken::green_key` and - // every `rd_loop_token` stamped off it must inherit the CELL - // key, or they name a cell the typed door never returns. - // `warmstate.py raise EnterJitAssembler(procedure_token, - // *execute_args)` carries the token read off the resolved cell - // onward for the same reason, rather than a number to re-derive - // it from. + // `force_start_tracing_for_key` set JC_TRACING on the cell + // selected by comparekey. Carry that cell's identity into the + // TraceCtx, compiled_loops/JitCellToken and the unconditional + // finally-clear instead of falling back to the bucket hash. + // This is the function-entry twin of on_back_edge_typed's + // resolve-once step below (`warmstate.py maybe_compile_and_run`). let green_key = Self::with_typed_decision_key(green_key, green_key_raw, |key| { self.warm_state.cell_key_for(key) }) .flatten() .unwrap_or(green_key); - // RPython pyjitpl.py create_empty_history(inputargs): the + self.prepare_trace_start_runtime(); + // RPython pyjitpl.py `create_empty_history(inputargs)`: the // MetaInterp owns the history/Trace factory, not warmstate. let mut recorder = crate::recorder::Trace::new(); for value in live_values { @@ -4848,7 +4844,26 @@ impl MetaInterp { } } - /// RPython warmstate.py bound_reached parity. + /// `warmstate.py maybe_compile_and_run`, function-threshold half. + /// Resolve the portal greens through `comparekey` and run the whole warm + /// decision on that cell. The returned boolean only says whether the + /// already-counted entry should call [`Self::force_start_tracing`]; that + /// call resolves the same typed key again without allocating on the common + /// unchained path. + pub fn should_trace_function_entry( + &mut self, + green_key: u64, + green_key_raw: (usize, usize), + ) -> bool { + match Self::with_typed_decision_key(green_key, green_key_raw, |key| { + self.warm_state.should_trace_function_entry_for_key(key) + }) { + Some(should_trace) => should_trace, + None => self.warm_state.should_trace_function_entry(green_key), + } + } + + /// RPython warmstate.py `bound_reached` parity. /// /// Like `on_back_edge_typed` but bypasses the counter tick — the /// caller (can_enter_jit_hook) already verified the counter fired. @@ -4878,6 +4893,11 @@ impl MetaInterp { match hot { HotResult::NotHot => BackEdgeAction::Interpret, HotResult::StartTracing => { + let green_key = Self::with_typed_decision_key(green_key, green_key_raw, |key| { + self.warm_state.cell_key_for(key) + }) + .flatten() + .unwrap_or(green_key); self.prepare_trace_start_runtime(); // Same re-resolve as the sibling `force_start_tracing` arm and // as `on_back_edge_typed`: the incoming number is a bucket @@ -7466,8 +7486,8 @@ impl MetaInterp { }; match compile_result { Ok(_) => { - self.last_compiled_artifact_invalidation_flag = Some(token.invalidation_flag()); - // compile.py store_hash: assign jitcounter hashes. + self.last_compiled_artifact_token = Some(token.clone()); + // compile.py `store_hash`: assign jitcounter hashes. self.assign_guard_hashes(token.as_ref()); // compile.py send_loop_to_backend registers the token // with the memory manager before record_loop_or_bridge reads it. @@ -8847,7 +8867,7 @@ impl MetaInterp { }; match compile_result { Ok(_) => { - self.last_compiled_artifact_invalidation_flag = Some(token.invalidation_flag()); + self.last_compiled_artifact_token = Some(token.clone()); self.assign_guard_hashes(token.as_ref()); // `compile.py` `propagate_original_jitcell_token(new_loop)`, // whose body at `:463-468` walks the trace's LABELs and sets @@ -9172,9 +9192,8 @@ impl MetaInterp { match result { Ok(_) => { - self.last_compiled_artifact_invalidation_flag = - source_jct.latest_bridge_invalidation_flag(); - // compile.py store_hash for bridge guards. + self.last_compiled_artifact_token = Some(source_jct.clone()); + // compile.py `store_hash` for bridge guards. self.assign_bridge_guard_hashes(source_jct.as_ref(), source_trace_id, fail_index); // `compile.py propagate_original_jitcell_token` — every // LABEL's TargetToken in the finished trace is rebound to @@ -9777,7 +9796,7 @@ impl MetaInterp { let compile_time = Instant::now().saturating_duration_since(compile_start); match compile_loop_result { Ok(_) => { - self.last_compiled_artifact_invalidation_flag = Some(token.invalidation_flag()); + self.last_compiled_artifact_token = Some(token.clone()); self.assign_guard_hashes(token.as_ref()); self.warm_state.memory_manager.keep_loop_alive(&token); // compile.py record_loop_or_bridge. @@ -10196,9 +10215,9 @@ impl MetaInterp { let compile_time = Instant::now().saturating_duration_since(compile_start); match compile_loop_result { Ok(_) => { - // compile.py record_loop_or_bridge registers every - // dependency against the artifact published by this compile. - self.last_compiled_artifact_invalidation_flag = Some(token.invalidation_flag()); + // compile.py `record_loop_or_bridge` registers every + // dependency against the owning token published by this compile. + self.last_compiled_artifact_token = Some(token.clone()); if !self.last_quasi_immutable_deps.is_empty() { crate::mc_diag_bump(75); } @@ -10404,17 +10423,15 @@ impl MetaInterp { self.last_compiled_key } - /// Flag read by `GUARD_NOT_INVALIDATED` in the last successful artifact. - pub fn last_compiled_artifact_invalidation_flag( - &self, - ) -> Option> { - self.last_compiled_artifact_invalidation_flag.clone() + /// Owning loop token of the last successful loop/bridge artifact. + pub fn last_compiled_artifact_token(&self) -> Option> { + self.last_compiled_artifact_token.clone() } - /// The flag names the artifact this compilation published, so a new + /// The token names the artifact owner this compilation published, so a new /// compilation attempt starts without one. - pub fn clear_last_compiled_artifact_invalidation_flag(&mut self) { - self.last_compiled_artifact_invalidation_flag = None; + pub fn clear_last_compiled_artifact_token(&mut self) { + self.last_compiled_artifact_token = None; } /// Cranelift direct body-entry selector for the first compiled loop LABEL. @@ -11882,9 +11899,10 @@ impl MetaInterp { // as `compile.py:204-207`. What keeps it out of this method is no // longer the dependency type (each entry is a // `majit_ir::QuasiImmutHandle`, which this crate can name) but - // upstream's `wref`: pyre's stand-in for the loop token is the - // artifact's invalidation flag, and the compiling driver owns it, not - // the metainterp. `last_quasi_immutable_deps` is the pyre-side analog + // upstream's `wref`: the concrete interpreter-side registration call + // lives in pyre-jit while the metainterp publishes the exact owning + // JitCellToken through `last_compiled_artifact_token`. + // `last_quasi_immutable_deps` is the pyre-side analog // of `loop.quasi_immutable_deps`, populated in this file from // `optimizer.quasi_immutable_deps` and drained by that walk. @@ -12919,9 +12937,9 @@ impl MetaInterp { match compile_result { Ok(_) => { - // compile.py record_loop_or_bridge registers every - // dependency against the artifact published by this compile. - self.last_compiled_artifact_invalidation_flag = Some(token.invalidation_flag()); + // compile.py `record_loop_or_bridge` registers every + // dependency against the owning token published by this compile. + self.last_compiled_artifact_token = Some(token.clone()); self.assign_guard_hashes(token.as_ref()); self.warm_state.memory_manager.keep_loop_alive(&token); // compile.py record_loop_or_bridge. @@ -13185,7 +13203,7 @@ impl MetaInterp { snapshot_frame_pcs: SnapshotFramePcs, call_pure_results: indexmap::IndexMap, Value>, ) -> bool { - self.last_compiled_artifact_invalidation_flag = None; + self.last_compiled_artifact_token = None; crate::mc_diag_bump(8); // compile_bridge entered if !self.compiled_loops.contains_key(&green_key) { return false; @@ -13759,8 +13777,7 @@ impl MetaInterp { match result { Ok(_) => { - self.last_compiled_artifact_invalidation_flag = - source_jct.latest_bridge_invalidation_flag(); + self.last_compiled_artifact_token = Some(source_jct.clone()); if crate::majit_log_enabled() { eprintln!( "[jit] compiled bridge at key={}, guard={}", @@ -23914,12 +23931,6 @@ mod tests { VALUES.get_or_init(|| Mutex::new(Vec::new())) } - #[allow(dead_code)] - fn may_force_test_lock() -> &'static Mutex<()> { - static LOCK: OnceLock> = OnceLock::new(); - LOCK.get_or_init(|| Mutex::new(())) - } - #[cfg(feature = "cranelift")] fn with_forced_deadframe(force_token: i64, f: impl FnOnce(DeadFrame)) { f(force_token_to_dead_frame(GcRef(force_token as usize))); @@ -24454,7 +24465,7 @@ mod tests { .collect() } - #[allow(dead_code)] + #[test] fn finish_trace_for_parity_preserves_captured_snapshots() { let mut meta = MetaInterp::<()>::new(10); meta.finish_setup_descrs_for_jitdrivers(); @@ -25519,6 +25530,51 @@ mod tests { ); } + #[test] + fn force_start_carries_the_typed_cells_minted_key_into_the_trace() { + let mut meta = MetaInterp::<()>::new(1); + meta.finish_setup_descrs_for_jitdrivers(); + let code: usize = 0x5100; + let pc: usize = 17; + let bucket = crate::green_key_from_code_ptr(code, pc); + let key = majit_ir::GreenKey::with_types( + vec![pc as i64, 0, code as i64], + vec![Type::Int, Type::Int, Type::Ref], + ); + assert_eq!(key.get_uhash(), bucket); + + // A hash-only writer occupies the raw bucket first, forcing the typed + // cell to receive a minted identity. The tracing session must carry + // that identity; otherwise its finally-clear and compiled token attach + // are redirected to the comparator-less head. + meta.warm_state.disable_noninlinable_function(bucket); + assert!(matches!( + meta.force_start_tracing(bucket, (code, pc), None, &[Value::Int(0)]), + BackEdgeAction::StartedTracing + )); + + let typed_cell_key = meta + .warm_state + .cell_key_for(&key) + .expect("force-start installed the typed cell"); + assert_ne!(typed_cell_key, bucket, "typed cell must be minted"); + assert_eq!( + meta.starting_green_key(), + Some(typed_cell_key), + "TraceCtx carries the resolved cell identity" + ); + + meta.warm_state.clear_tracing_flag(typed_cell_key); + assert!( + !meta + .warm_state + .lookup_chain_with_key(&key) + .expect("typed cell") + .is_tracing(), + "the identity carried by the trace clears the cell it started" + ); + } + #[test] fn test_on_compile_error_fires_on_failure() { // Parity with test_on_abort: on_compile_error fires when compilation fails. diff --git a/majit/majit-metainterp/src/recorder.rs b/majit/majit-metainterp/src/recorder.rs index ac8bf72d82a..f12ff3e9ccb 100644 --- a/majit/majit-metainterp/src/recorder.rs +++ b/majit/majit-metainterp/src/recorder.rs @@ -181,6 +181,12 @@ pub struct Trace { /// opencoder.py parity: count of box-yielding positions /// (inputargs + non-void ops). box_count: u32, + /// Monotonic count of operations appended during this recording session. + /// Unlike `ops.len()`, this is not rewound by [`Self::cut`]. The bridge + /// driver uses it to distinguish an abort before the body walk from a + /// body abort whose speculative operations were cut back to the setup + /// position (`history.cut` in `pyjitpl.py`). + recorded_ops_total: usize, } impl Trace { @@ -196,6 +202,7 @@ impl Trace { op_count: 0, guard_count: 0, box_count: 0, + recorded_ops_total: 0, } } @@ -324,6 +331,7 @@ impl Trace { let op = Op::new(opcode, &self.box_args(args)); op.pos.set(opref); self.ops.push(OpRc::new(op)); + self.recorded_ops_total += 1; self.op_count += 1; if opcode.result_type() != Type::Void { self.box_count += 1; @@ -344,6 +352,7 @@ impl Trace { let op = Op::with_descr(opcode, &self.box_args(args), descr); op.pos.set(opref); self.ops.push(OpRc::new(op)); + self.recorded_ops_total += 1; self.op_count += 1; if opcode.result_type() != Type::Void { self.box_count += 1; @@ -373,6 +382,7 @@ impl Trace { }; op.pos.set(opref); self.ops.push(OpRc::new(op)); + self.recorded_ops_total += 1; self.op_count += 1; if opcode.result_type() != Type::Void { self.box_count += 1; @@ -400,6 +410,7 @@ impl Trace { op.pos.set(opref); op.setfailargs(self.box_args(fail_args).iter().cloned().collect()); self.ops.push(OpRc::new(op)); + self.recorded_ops_total += 1; self.op_count += 1; if opcode.result_type() != Type::Void { self.box_count += 1; @@ -554,6 +565,7 @@ impl Trace { }; op.pos.set(opref); self.ops.push(OpRc::new(op)); + self.recorded_ops_total += 1; self.op_count += 1; if OpCode::Jump.result_type() != Type::Void { self.box_count += 1; @@ -567,6 +579,7 @@ impl Trace { let op = Op::with_descr(OpCode::Finish, &self.box_args(finish_args), descr); op.pos.set(opref); self.ops.push(OpRc::new(op)); + self.recorded_ops_total += 1; self.op_count += 1; if OpCode::Finish.result_type() != Type::Void { self.box_count += 1; @@ -633,6 +646,12 @@ impl Trace { self.ops.len() } + /// Number of operations ever appended to this recorder, including + /// speculative operations subsequently discarded by [`Self::cut`]. + pub fn recorded_ops_total(&self) -> usize { + self.recorded_ops_total + } + /// Number of input arguments registered. pub fn num_inputargs(&self) -> usize { self.inputargs.len() @@ -813,6 +832,7 @@ mod tests { assert_eq!(rec.num_ops(), 2); rec.cut(saved); assert_eq!(rec.num_ops(), 0); + assert_eq!(rec.recorded_ops_total(), 2); assert_eq!(rec.num_inputargs(), 1); } diff --git a/majit/majit-metainterp/src/resume.rs b/majit/majit-metainterp/src/resume.rs index 094b4677816..0b91583a52f 100644 --- a/majit/majit-metainterp/src/resume.rs +++ b/majit/majit-metainterp/src/resume.rs @@ -641,18 +641,6 @@ impl ResumeStorage { self.rd_consts.as_slice() } - /// Internal accessor for the GC root walker. SAFETY: caller must - /// ensure exclusive access — only the minor-collection walker in - /// `MetaInterp::walk_rd_consts_refs` uses this. - #[expect( - clippy::mut_from_ref, - reason = "The stop-the-world GC root walker is the sole writer and ResumeDataStorage owns an UnsafeCell-backed root vector; this unsafe accessor makes that externally enforced exclusivity explicit" - )] - #[allow(dead_code)] - pub(crate) unsafe fn rd_consts_mut_for_gc(&self) -> &mut Vec { - unsafe { self.rd_consts.as_mut_vec_for_gc() } - } - pub fn with_shared_consts( rd_numb: Vec, rd_consts: Arc, diff --git a/majit/majit-metainterp/src/trace_ctx.rs b/majit/majit-metainterp/src/trace_ctx.rs index d7e938adcae..86b2cbb3fbb 100644 --- a/majit/majit-metainterp/src/trace_ctx.rs +++ b/majit/majit-metainterp/src/trace_ctx.rs @@ -2244,6 +2244,13 @@ impl TraceCtx { self.recorder.num_ops() } + /// Monotonic counterpart to [`Self::num_ops`]. `history.cut` can restore + /// the current length, but cannot make a body walk that already ran into a + /// setup-time abort again. + pub fn recorded_ops_total(&self) -> usize { + self.recorder.recorded_ops_total() + } + /// Diagnostic: dump every recorded op (result OpRef = pos, opcode, args) /// to stderr. Used by the P2 carrier investigation to inspect the /// def-use of the fused (callee continuation + root) bridge trace — diff --git a/majit/majit-metainterp/src/warmstate.rs b/majit/majit-metainterp/src/warmstate.rs index 47c79bc4631..1bec538be22 100644 --- a/majit/majit-metainterp/src/warmstate.rs +++ b/majit/majit-metainterp/src/warmstate.rs @@ -1162,7 +1162,16 @@ impl WarmEnterState { if cell.is_tracing() { return HotResult::AlreadyTracing; } - if cell.flags & jc_flags::DONT_TRACE_HERE != 0 && cell.has_seen_a_procedure_token() { + // `warmstate.py maybe_compile_and_run` handles JC_TEMPORARY before it consults + // JC_DONT_TRACE_HERE: a compile_tmp_callback is only the residual + // CALL_ASSEMBLER fallback, so it must keep counting toward the + // callee's real standalone trace even when that callee was marked + // non-inlinable. Refusing the combination here strands the + // JC_FORCE_FINISH retry installed after a trace-too-long abort. + if cell.flags & jc_flags::DONT_TRACE_HERE != 0 + && cell.flags & jc_flags::TEMPORARY == 0 + && cell.has_seen_a_procedure_token() + { return HotResult::NotHot; } // Give up after too many failed trace attempts to prevent @@ -1189,7 +1198,10 @@ impl WarmEnterState { if cell.is_tracing() { return HotResult::AlreadyTracing; } - if cell.flags & jc_flags::DONT_TRACE_HERE != 0 && cell.has_seen_a_procedure_token() { + if cell.flags & jc_flags::DONT_TRACE_HERE != 0 + && cell.flags & jc_flags::TEMPORARY == 0 + && cell.has_seen_a_procedure_token() + { return HotResult::NotHot; } if cell.abort_count >= MAX_TRACE_ABORT_COUNT { @@ -1874,6 +1886,19 @@ impl WarmEnterState { .is_none_or(|cell| cell.flags & jc_flags::DONT_TRACE_HERE == 0) } + /// Typed-key variant of [`Self::can_inline_callable`]. + /// + /// `warmstate.py` `can_inline_callable` receives the portal greens and + /// reaches the `JitCell` through `get_jitcell_at_key`, whose chain walk + /// calls `comparekey`. A producer that still has those greens must use + /// this door: reading the bucket's hash-form cell can miss the + /// `DONT_TRACE_HERE` flag that `dont_trace_here(*greenargs)` set on the + /// typed cell. + pub fn can_inline_callable_for_key(&self, key: &GreenKey) -> bool { + self.lookup_chain_with_key(key) + .is_none_or(|cell| cell.flags & jc_flags::DONT_TRACE_HERE == 0) + } + /// Mark a callee as a location that should no longer be inlined into /// surrounding traces. /// @@ -2075,12 +2100,25 @@ impl WarmEnterState { crate::mc_diag_bump(81); // abort_ceiling_refused return false; } + // `warmstate.py maybe_compile_and_run`: JC_TEMPORARY is tested alongside + // JC_TRACING and, unlike JC_TRACING, counts normally. This branch + // must precede JC_DONT_TRACE_HERE below: the temporary token is the + // interpreter callback used until the non-inlinable callee gets + // its own real trace, not evidence that the callee was already + // compiled separately. + if cell.flags & jc_flags::TEMPORARY != 0 { + crate::mc_diag_bump(25); + return self + .counter + .tick(self.bucket_of(cell_key), self.increment_function_threshold); + } if cell.flags & jc_flags::DONT_TRACE_HERE != 0 { if cell.has_seen_a_procedure_token() { - // A live TEMPORARY token still declines; a token that was - // once seen but has since been invalidated falls through to - // the cleanup gate below (warmstate.py:483-491) rather than - // re-entering the never-traced retry. + // A non-temporary token that was once seen but has since + // been invalidated falls through to the cleanup gate below + // (`warmstate.py maybe_compile_and_run`) rather than re-entering the + // never-traced retry. A live real token returned through + // the compiled branch above. if cell.get_procedure_token().is_some() { return false; } @@ -2104,6 +2142,60 @@ impl WarmEnterState { .tick(self.bucket_of(cell_key), self.increment_function_threshold) } + /// Typed-key form of [`Self::should_trace_function_entry`]. + /// + /// `warmstate.py maybe_compile_and_run` walks the bucket chain + /// with `comparekey` once and keeps that exact `JitCell` through the + /// JC_TEMPORARY / procedure-token / JC_DONT_TRACE_HERE decision. A raw + /// hash names only the bucket in pyre, so using the hash form for this + /// first half and [`Self::force_start_tracing_for_key`] for the second can + /// inspect two different cells in a chained bucket. Keep the whole + /// function-entry decision on the matching typed cell instead. + pub fn should_trace_function_entry_for_key(&mut self, key: &GreenKey) -> bool { + let bucket = key.get_uhash(); + let mut cleanup_dead_token_cell = false; + if let Some(cell) = self.lookup_chain_with_key(key) { + let compiled = cell.is_compiled(); + let tracing = cell.is_tracing(); + if compiled || tracing { + crate::mc_diag_bump(23); + if compiled { + crate::mc_diag_bump(64); + } + if tracing { + crate::mc_diag_bump(65); + if cell.tracing_generation < self.tracing_generation { + crate::mc_diag_bump(66); + } + } + return false; + } + if cell.flags & jc_flags::TEMPORARY != 0 { + crate::mc_diag_bump(25); + return self.counter.tick(bucket, self.increment_function_threshold); + } + if cell.flags & jc_flags::DONT_TRACE_HERE != 0 { + if cell.has_seen_a_procedure_token() { + if cell.get_procedure_token().is_some() { + return false; + } + } else if cell.flags & jc_flags::TRACING_OCCURRED == 0 { + return true; + } + } + if cell.has_seen_a_procedure_token() && cell.get_procedure_token().is_none() { + cleanup_dead_token_cell = true; + } + } + if cleanup_dead_token_cell { + crate::mc_diag_bump(24); + self.cleanup_chain(bucket); + return false; + } + crate::mc_diag_bump(25); + self.counter.tick(bucket, self.increment_function_threshold) + } + /// Check if inlining is allowed at the given depth. pub fn can_inline_at_depth(&self, current_depth: usize) -> bool { (current_depth as u32) < self.max_inline_depth @@ -3556,6 +3648,16 @@ mod tests { assert!(!ws.can_inline_callable(42)); } + #[test] + fn typed_disable_blocks_the_same_typed_inline_lookup() { + let mut ws = WarmEnterState::new(3); + let key = GreenKey::new(vec![7, 11]); + + assert!(ws.can_inline_callable_for_key(&key)); + ws.disable_noninlinable_function_for_key(&key); + assert!(!ws.can_inline_callable_for_key(&key)); + } + #[test] fn test_abort_too_long_then_retry_different_key() { // Aborting one key's trace as too long should not affect other keys. @@ -5139,6 +5241,58 @@ mod tests { ); } + /// `warmstate.py maybe_compile_and_run` checks JC_TEMPORARY before + /// JC_DONT_TRACE_HERE. A callee blamed for a trace-too-long abort can + /// carry all of TEMPORARY (its residual CALL_ASSEMBLER callback), + /// DONT_TRACE_HERE (do not inline it again), and FORCE_FINISH (segment its + /// standalone trace). That combination must count and then start tracing + /// the matching typed cell; otherwise the callee can never replace its + /// temporary callback with a real loop. + #[test] + fn temporary_noninlinable_function_entry_traces_its_typed_cell() { + let mut ws = WarmEnterState::new(100); + ws.set_function_threshold(1); + let key = GreenKey::new(vec![30, 40]); + let bucket = key.get_uhash(); + + // Put a comparator-less cell at the bucket head first. This is the + // production migration shape in which a raw-hash precheck and a typed + // force-start used to inspect different cells for one green key. + ws.disable_noninlinable_function(bucket); + let token = std::sync::Arc::new(JitCellToken::new(0xcafe)); + ws.get_assembler_token_with_key::<(), _>(&key, |_memmgr| Ok(token.clone())) + .expect("temporary token install"); + ws.disable_noninlinable_function_for_key(&key); + ws.mark_force_finish_tracing_for_key(&key); + + let typed = ws.lookup_chain_with_key(&key).expect("typed callee cell"); + assert_ne!(typed.cell_key, Some(bucket), "typed cell is chained/minted"); + assert_ne!(typed.flags & jc_flags::TEMPORARY, 0); + assert_ne!(typed.flags & jc_flags::DONT_TRACE_HERE, 0); + assert_ne!(typed.flags & jc_flags::FORCE_FINISH, 0); + + assert!( + ws.should_trace_function_entry_for_key(&key), + "JC_TEMPORARY counts normally despite DONT_TRACE_HERE" + ); + assert!(matches!( + ws.force_start_tracing_for_key(&key), + HotResult::StartTracing + )); + + let typed = ws.lookup_chain_with_key(&key).expect("typed callee cell"); + assert!(typed.is_tracing(), "the matching typed cell starts tracing"); + assert_ne!( + typed.flags & jc_flags::FORCE_FINISH, + 0, + "the standalone retry retains its segmenting request" + ); + assert!( + !ws.cell_by_key(bucket).expect("hash-only head").is_tracing(), + "the comparator-less bucket head is not mistaken for the callee" + ); + } + /// `warmstate.py:714-723` + `626-641` — typed variant must /// disambiguate hash collisions: two `GreenKey`s that share a hash /// but compare unequal under `equal_whatever` get distinct tokens. diff --git a/pyre/bench/fib_recursive.cranelift.jitstats b/pyre/bench/fib_recursive.cranelift.jitstats index 09180214fa8..80953cf4528 100644 --- a/pyre/bench/fib_recursive.cranelift.jitstats +++ b/pyre/bench/fib_recursive.cranelift.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1645 +guard_failures=1600 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 diff --git a/pyre/bench/fib_recursive.dynasm.jitstats b/pyre/bench/fib_recursive.dynasm.jitstats index 09180214fa8..80953cf4528 100644 --- a/pyre/bench/fib_recursive.dynasm.jitstats +++ b/pyre/bench/fib_recursive.dynasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1645 +guard_failures=1600 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 diff --git a/pyre/bench/fib_recursive.wasm.jitstats b/pyre/bench/fib_recursive.wasm.jitstats index 09180214fa8..80953cf4528 100644 --- a/pyre/bench/fib_recursive.wasm.jitstats +++ b/pyre/bench/fib_recursive.wasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1645 +guard_failures=1600 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 diff --git a/pyre/bench/synth/bridge_global_fold_invalidate_hot.cranelift.jitstats b/pyre/bench/synth/bridge_global_fold_invalidate_hot.cranelift.jitstats index 80e7f9e6142..56b36cc0db2 100644 --- a/pyre/bench/synth/bridge_global_fold_invalidate_hot.cranelift.jitstats +++ b/pyre/bench/synth/bridge_global_fold_invalidate_hot.cranelift.jitstats @@ -12,3 +12,4 @@ guard_failures=1150 internal_compile_panics=0 loops_aborted=0 loops_compiled=9 +retraces_compiled=0 diff --git a/pyre/bench/synth/bridge_global_fold_invalidate_hot.dynasm.jitstats b/pyre/bench/synth/bridge_global_fold_invalidate_hot.dynasm.jitstats index 80e7f9e6142..56b36cc0db2 100644 --- a/pyre/bench/synth/bridge_global_fold_invalidate_hot.dynasm.jitstats +++ b/pyre/bench/synth/bridge_global_fold_invalidate_hot.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=1150 internal_compile_panics=0 loops_aborted=0 loops_compiled=9 +retraces_compiled=0 diff --git a/pyre/bench/synth/bridge_global_fold_invalidate_hot.wasm.jitstats b/pyre/bench/synth/bridge_global_fold_invalidate_hot.wasm.jitstats index 80e7f9e6142..56b36cc0db2 100644 --- a/pyre/bench/synth/bridge_global_fold_invalidate_hot.wasm.jitstats +++ b/pyre/bench/synth/bridge_global_fold_invalidate_hot.wasm.jitstats @@ -12,3 +12,4 @@ guard_failures=1150 internal_compile_panics=0 loops_aborted=0 loops_compiled=9 +retraces_compiled=0 diff --git a/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats b/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats index 241a18080cf..0849e7d763f 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats +++ b/pyre/bench/synth/bridge_recursion_overflow.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=962 +guard_failures=626 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats b/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats index 241a18080cf..0849e7d763f 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats +++ b/pyre/bench/synth/bridge_recursion_overflow.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=962 +guard_failures=626 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats b/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats index 241a18080cf..0849e7d763f 100644 --- a/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats +++ b/pyre/bench/synth/bridge_recursion_overflow.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=3 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=962 +guard_failures=626 internal_compile_panics=0 loops_aborted=0 loops_compiled=1 +retraces_compiled=0 diff --git a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats index 2d7cef88ecd..6ff1d10bc19 100644 --- a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats +++ b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=24 +bridges_compiled=25 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3369 +guard_failures=4607 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats index 2d7cef88ecd..6ff1d10bc19 100644 --- a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats +++ b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=24 +bridges_compiled=25 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3369 +guard_failures=4607 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats index 2d7cef88ecd..6ff1d10bc19 100644 --- a/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats +++ b/pyre/bench/synth/ca_bridge_multiframe_resume_double_call.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=24 +bridges_compiled=25 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3369 +guard_failures=4607 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/calls_closures.cranelift.jitstats b/pyre/bench/synth/calls_closures.cranelift.jitstats index 3e5329580ca..80effc54024 100644 --- a/pyre/bench/synth/calls_closures.cranelift.jitstats +++ b/pyre/bench/synth/calls_closures.cranelift.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1896 +guard_failures=1818 internal_compile_panics=0 loops_aborted=0 loops_compiled=8 diff --git a/pyre/bench/synth/calls_closures.dynasm.jitstats b/pyre/bench/synth/calls_closures.dynasm.jitstats index 3e5329580ca..80effc54024 100644 --- a/pyre/bench/synth/calls_closures.dynasm.jitstats +++ b/pyre/bench/synth/calls_closures.dynasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1896 +guard_failures=1818 internal_compile_panics=0 loops_aborted=0 loops_compiled=8 diff --git a/pyre/bench/synth/calls_closures.wasm.jitstats b/pyre/bench/synth/calls_closures.wasm.jitstats index 3e5329580ca..80effc54024 100644 --- a/pyre/bench/synth/calls_closures.wasm.jitstats +++ b/pyre/bench/synth/calls_closures.wasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1896 +guard_failures=1818 internal_compile_panics=0 loops_aborted=0 loops_compiled=8 diff --git a/pyre/bench/synth/closure_freevar_branch_resume.cranelift.jitstats b/pyre/bench/synth/closure_freevar_branch_resume.cranelift.jitstats index 17b3b744da4..ade70e068aa 100644 --- a/pyre/bench/synth/closure_freevar_branch_resume.cranelift.jitstats +++ b/pyre/bench/synth/closure_freevar_branch_resume.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=606 +guard_failures=1226 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=6 +retraces_compiled=0 diff --git a/pyre/bench/synth/closure_freevar_branch_resume.dynasm.jitstats b/pyre/bench/synth/closure_freevar_branch_resume.dynasm.jitstats index 17b3b744da4..ade70e068aa 100644 --- a/pyre/bench/synth/closure_freevar_branch_resume.dynasm.jitstats +++ b/pyre/bench/synth/closure_freevar_branch_resume.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=606 +guard_failures=1226 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=6 +retraces_compiled=0 diff --git a/pyre/bench/synth/closure_freevar_branch_resume.wasm.jitstats b/pyre/bench/synth/closure_freevar_branch_resume.wasm.jitstats index 17b3b744da4..ade70e068aa 100644 --- a/pyre/bench/synth/closure_freevar_branch_resume.wasm.jitstats +++ b/pyre/bench/synth/closure_freevar_branch_resume.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=606 +guard_failures=1226 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=6 +retraces_compiled=0 diff --git a/pyre/bench/synth/del_cellvar_walk_commit.cranelift.jitstats b/pyre/bench/synth/del_cellvar_walk_commit.cranelift.jitstats index 59f22855e15..4e1e41a7bed 100644 --- a/pyre/bench/synth/del_cellvar_walk_commit.cranelift.jitstats +++ b/pyre/bench/synth/del_cellvar_walk_commit.cranelift.jitstats @@ -11,4 +11,5 @@ field_pos_spec_misplaced=0 guard_failures=1 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/del_cellvar_walk_commit.dynasm.jitstats b/pyre/bench/synth/del_cellvar_walk_commit.dynasm.jitstats index 59f22855e15..4e1e41a7bed 100644 --- a/pyre/bench/synth/del_cellvar_walk_commit.dynasm.jitstats +++ b/pyre/bench/synth/del_cellvar_walk_commit.dynasm.jitstats @@ -11,4 +11,5 @@ field_pos_spec_misplaced=0 guard_failures=1 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/del_cellvar_walk_commit.wasm.jitstats b/pyre/bench/synth/del_cellvar_walk_commit.wasm.jitstats index 59f22855e15..4e1e41a7bed 100644 --- a/pyre/bench/synth/del_cellvar_walk_commit.wasm.jitstats +++ b/pyre/bench/synth/del_cellvar_walk_commit.wasm.jitstats @@ -11,4 +11,5 @@ field_pos_spec_misplaced=0 guard_failures=1 internal_compile_panics=0 loops_aborted=0 -loops_compiled=1 +loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.cranelift.jitstats b/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.cranelift.jitstats index 592bd139af0..988daba377e 100644 --- a/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.cranelift.jitstats +++ b/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=807 +guard_failures=1407 internal_compile_panics=0 loops_aborted=0 -loops_compiled=16 +loops_compiled=17 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.dynasm.jitstats b/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.dynasm.jitstats index 592bd139af0..988daba377e 100644 --- a/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.dynasm.jitstats +++ b/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=807 +guard_failures=1407 internal_compile_panics=0 loops_aborted=0 -loops_compiled=16 +loops_compiled=17 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.wasm.jitstats b/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.wasm.jitstats index 592bd139af0..988daba377e 100644 --- a/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.wasm.jitstats +++ b/pyre/bench/synth/exception_escape_hot_callee_tb_node_once.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=4 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=807 +guard_failures=1407 internal_compile_panics=0 loops_aborted=0 -loops_compiled=16 +loops_compiled=17 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_reentry_guard_finally_residual.cranelift.jitstats b/pyre/bench/synth/exception_reentry_guard_finally_residual.cranelift.jitstats index 2f7eeec1bd3..5ec6a1f109d 100644 --- a/pyre/bench/synth/exception_reentry_guard_finally_residual.cranelift.jitstats +++ b/pyre/bench/synth/exception_reentry_guard_finally_residual.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=12 +bridges_compiled=5 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2461 +guard_failures=1026 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=6 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_reentry_guard_finally_residual.dynasm.jitstats b/pyre/bench/synth/exception_reentry_guard_finally_residual.dynasm.jitstats index 2f7eeec1bd3..5ec6a1f109d 100644 --- a/pyre/bench/synth/exception_reentry_guard_finally_residual.dynasm.jitstats +++ b/pyre/bench/synth/exception_reentry_guard_finally_residual.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=12 +bridges_compiled=5 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2461 +guard_failures=1026 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=6 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_reentry_guard_finally_residual.wasm.jitstats b/pyre/bench/synth/exception_reentry_guard_finally_residual.wasm.jitstats index 2f7eeec1bd3..0de675fbeac 100644 --- a/pyre/bench/synth/exception_reentry_guard_finally_residual.wasm.jitstats +++ b/pyre/bench/synth/exception_reentry_guard_finally_residual.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=12 +bridges_compiled=5 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2461 +guard_failures=1022 internal_compile_panics=0 loops_aborted=0 -loops_compiled=3 +loops_compiled=6 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_traceback_frame_lineno.cranelift.jitstats b/pyre/bench/synth/exception_traceback_frame_lineno.cranelift.jitstats index 7b03db0478f..ff83064a6ba 100644 --- a/pyre/bench/synth/exception_traceback_frame_lineno.cranelift.jitstats +++ b/pyre/bench/synth/exception_traceback_frame_lineno.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=5 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=812 +guard_failures=1012 internal_compile_panics=0 loops_aborted=0 -loops_compiled=17 +loops_compiled=18 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_traceback_frame_lineno.dynasm.jitstats b/pyre/bench/synth/exception_traceback_frame_lineno.dynasm.jitstats index 7b03db0478f..ff83064a6ba 100644 --- a/pyre/bench/synth/exception_traceback_frame_lineno.dynasm.jitstats +++ b/pyre/bench/synth/exception_traceback_frame_lineno.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=5 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=812 +guard_failures=1012 internal_compile_panics=0 loops_aborted=0 -loops_compiled=17 +loops_compiled=18 +retraces_compiled=0 diff --git a/pyre/bench/synth/exception_traceback_frame_lineno.wasm.jitstats b/pyre/bench/synth/exception_traceback_frame_lineno.wasm.jitstats index 7b03db0478f..ff83064a6ba 100644 --- a/pyre/bench/synth/exception_traceback_frame_lineno.wasm.jitstats +++ b/pyre/bench/synth/exception_traceback_frame_lineno.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=5 +bridges_compiled=6 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=812 +guard_failures=1012 internal_compile_panics=0 loops_aborted=0 -loops_compiled=17 +loops_compiled=18 +retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats b/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats index 6ead156abb9..c2a293011b6 100644 --- a/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats +++ b/pyre/bench/synth/foriter_call_resume_drops_iteration.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=35 +bridges_compiled=33 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4784 +guard_failures=5239 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats b/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats index 6ead156abb9..c2a293011b6 100644 --- a/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats +++ b/pyre/bench/synth/foriter_call_resume_drops_iteration.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=35 +bridges_compiled=33 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4784 +guard_failures=5239 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats b/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats index 6ead156abb9..c2a293011b6 100644 --- a/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats +++ b/pyre/bench/synth/foriter_call_resume_drops_iteration.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=35 +bridges_compiled=33 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=4784 +guard_failures=5239 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/foriter_exempt_nested_foriter.cranelift.jitstats b/pyre/bench/synth/foriter_exempt_nested_foriter.cranelift.jitstats index f6ab8fc853d..cd85f0b233d 100644 --- a/pyre/bench/synth/foriter_exempt_nested_foriter.cranelift.jitstats +++ b/pyre/bench/synth/foriter_exempt_nested_foriter.cranelift.jitstats @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 loops_aborted=1 -loops_compiled=1 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_exempt_nested_foriter.dynasm.jitstats b/pyre/bench/synth/foriter_exempt_nested_foriter.dynasm.jitstats index f6ab8fc853d..cd85f0b233d 100644 --- a/pyre/bench/synth/foriter_exempt_nested_foriter.dynasm.jitstats +++ b/pyre/bench/synth/foriter_exempt_nested_foriter.dynasm.jitstats @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 loops_aborted=1 -loops_compiled=1 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_exempt_nested_foriter.wasm.jitstats b/pyre/bench/synth/foriter_exempt_nested_foriter.wasm.jitstats index f6ab8fc853d..cd85f0b233d 100644 --- a/pyre/bench/synth/foriter_exempt_nested_foriter.wasm.jitstats +++ b/pyre/bench/synth/foriter_exempt_nested_foriter.wasm.jitstats @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 loops_aborted=1 -loops_compiled=1 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_exempt_shared_generator.cranelift.jitstats b/pyre/bench/synth/foriter_exempt_shared_generator.cranelift.jitstats index f6ab8fc853d..cd85f0b233d 100644 --- a/pyre/bench/synth/foriter_exempt_shared_generator.cranelift.jitstats +++ b/pyre/bench/synth/foriter_exempt_shared_generator.cranelift.jitstats @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 loops_aborted=1 -loops_compiled=1 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_exempt_shared_generator.dynasm.jitstats b/pyre/bench/synth/foriter_exempt_shared_generator.dynasm.jitstats index f6ab8fc853d..cd85f0b233d 100644 --- a/pyre/bench/synth/foriter_exempt_shared_generator.dynasm.jitstats +++ b/pyre/bench/synth/foriter_exempt_shared_generator.dynasm.jitstats @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 loops_aborted=1 -loops_compiled=1 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/foriter_exempt_shared_generator.wasm.jitstats b/pyre/bench/synth/foriter_exempt_shared_generator.wasm.jitstats index f6ab8fc853d..cd85f0b233d 100644 --- a/pyre/bench/synth/foriter_exempt_shared_generator.wasm.jitstats +++ b/pyre/bench/synth/foriter_exempt_shared_generator.wasm.jitstats @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=0 +guard_failures=1 internal_compile_panics=0 loops_aborted=1 -loops_compiled=1 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats b/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats index 4068718b6be..ff956179270 100644 --- a/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats +++ b/pyre/bench/synth/generator_tree_recursion.cranelift.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2999 +guard_failures=3013 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats b/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats index 4068718b6be..ff956179270 100644 --- a/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats +++ b/pyre/bench/synth/generator_tree_recursion.dynasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2999 +guard_failures=3013 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/generator_tree_recursion.wasm.jitstats b/pyre/bench/synth/generator_tree_recursion.wasm.jitstats index 4068718b6be..ff956179270 100644 --- a/pyre/bench/synth/generator_tree_recursion.wasm.jitstats +++ b/pyre/bench/synth/generator_tree_recursion.wasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=2999 +guard_failures=3013 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/inline_subwalk_user_iterator.cranelift.jitstats b/pyre/bench/synth/inline_subwalk_user_iterator.cranelift.jitstats index 9e3a51a9dc9..3dfa066ff46 100644 --- a/pyre/bench/synth/inline_subwalk_user_iterator.cranelift.jitstats +++ b/pyre/bench/synth/inline_subwalk_user_iterator.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=1 +bridges_compiled=2 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=200 +guard_failures=401 internal_compile_panics=0 loops_aborted=1 -loops_compiled=2 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/inline_subwalk_user_iterator.dynasm.jitstats b/pyre/bench/synth/inline_subwalk_user_iterator.dynasm.jitstats index 9e3a51a9dc9..3dfa066ff46 100644 --- a/pyre/bench/synth/inline_subwalk_user_iterator.dynasm.jitstats +++ b/pyre/bench/synth/inline_subwalk_user_iterator.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=1 +bridges_compiled=2 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=200 +guard_failures=401 internal_compile_panics=0 loops_aborted=1 -loops_compiled=2 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/inline_subwalk_user_iterator.wasm.jitstats b/pyre/bench/synth/inline_subwalk_user_iterator.wasm.jitstats index 9e3a51a9dc9..3dfa066ff46 100644 --- a/pyre/bench/synth/inline_subwalk_user_iterator.wasm.jitstats +++ b/pyre/bench/synth/inline_subwalk_user_iterator.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=1 +bridges_compiled=2 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,8 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=200 +guard_failures=401 internal_compile_panics=0 loops_aborted=1 -loops_compiled=2 +loops_compiled=3 retraces_compiled=0 diff --git a/pyre/bench/synth/polymorphic_binary_receiver.cranelift.jitstats b/pyre/bench/synth/polymorphic_binary_receiver.cranelift.jitstats index 5a9d4752958..ea8233e5b72 100644 --- a/pyre/bench/synth/polymorphic_binary_receiver.cranelift.jitstats +++ b/pyre/bench/synth/polymorphic_binary_receiver.cranelift.jitstats @@ -12,3 +12,4 @@ guard_failures=788 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 +retraces_compiled=0 diff --git a/pyre/bench/synth/polymorphic_binary_receiver.dynasm.jitstats b/pyre/bench/synth/polymorphic_binary_receiver.dynasm.jitstats index 5a9d4752958..ea8233e5b72 100644 --- a/pyre/bench/synth/polymorphic_binary_receiver.dynasm.jitstats +++ b/pyre/bench/synth/polymorphic_binary_receiver.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=788 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 +retraces_compiled=0 diff --git a/pyre/bench/synth/polymorphic_binary_receiver.wasm.jitstats b/pyre/bench/synth/polymorphic_binary_receiver.wasm.jitstats index 5a9d4752958..ea8233e5b72 100644 --- a/pyre/bench/synth/polymorphic_binary_receiver.wasm.jitstats +++ b/pyre/bench/synth/polymorphic_binary_receiver.wasm.jitstats @@ -12,3 +12,4 @@ guard_failures=788 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 +retraces_compiled=0 diff --git a/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats b/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats index e2aa182c4fa..f2ba0448cf7 100644 --- a/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats +++ b/pyre/bench/synth/recursion_memo_branch.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=25 +bridges_compiled=31 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3491 +guard_failures=5048 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats b/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats index e2aa182c4fa..f2ba0448cf7 100644 --- a/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats +++ b/pyre/bench/synth/recursion_memo_branch.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=25 +bridges_compiled=31 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3491 +guard_failures=5048 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/recursion_memo_branch.wasm.jitstats b/pyre/bench/synth/recursion_memo_branch.wasm.jitstats index e2aa182c4fa..f2ba0448cf7 100644 --- a/pyre/bench/synth/recursion_memo_branch.wasm.jitstats +++ b/pyre/bench/synth/recursion_memo_branch.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=25 +bridges_compiled=31 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=3491 +guard_failures=5048 internal_compile_panics=0 loops_aborted=0 loops_compiled=3 diff --git a/pyre/bench/synth/recursive_forced_frame_kept_stack.cranelift.jitstats b/pyre/bench/synth/recursive_forced_frame_kept_stack.cranelift.jitstats index 69db3f4ee62..60a9330d6be 100644 --- a/pyre/bench/synth/recursive_forced_frame_kept_stack.cranelift.jitstats +++ b/pyre/bench/synth/recursive_forced_frame_kept_stack.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=2 +bridges_compiled=4 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=400 +guard_failures=800 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/recursive_forced_frame_kept_stack.dynasm.jitstats b/pyre/bench/synth/recursive_forced_frame_kept_stack.dynasm.jitstats index 69db3f4ee62..60a9330d6be 100644 --- a/pyre/bench/synth/recursive_forced_frame_kept_stack.dynasm.jitstats +++ b/pyre/bench/synth/recursive_forced_frame_kept_stack.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=2 +bridges_compiled=4 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=400 +guard_failures=800 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats b/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats index 9352e9e4f9b..1c0ba3e3f5b 100644 --- a/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats +++ b/pyre/bench/synth/selfrec_bridge_nontail_promote.cranelift.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=805 +guard_failures=606 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats b/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats index 9352e9e4f9b..1c0ba3e3f5b 100644 --- a/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats +++ b/pyre/bench/synth/selfrec_bridge_nontail_promote.dynasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=805 +guard_failures=606 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats b/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats index 9352e9e4f9b..1c0ba3e3f5b 100644 --- a/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats +++ b/pyre/bench/synth/selfrec_bridge_nontail_promote.wasm.jitstats @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=805 +guard_failures=606 internal_compile_panics=0 loops_aborted=0 loops_compiled=2 diff --git a/pyre/bench/synth/short_circuit_side_effects.cranelift.jitstats b/pyre/bench/synth/short_circuit_side_effects.cranelift.jitstats index fd8634285b9..7d816fe1165 100644 --- a/pyre/bench/synth/short_circuit_side_effects.cranelift.jitstats +++ b/pyre/bench/synth/short_circuit_side_effects.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=7 +bridges_compiled=8 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1972 +guard_failures=2104 internal_compile_panics=0 loops_aborted=0 loops_compiled=4 +retraces_compiled=0 diff --git a/pyre/bench/synth/short_circuit_side_effects.dynasm.jitstats b/pyre/bench/synth/short_circuit_side_effects.dynasm.jitstats index fd8634285b9..7d816fe1165 100644 --- a/pyre/bench/synth/short_circuit_side_effects.dynasm.jitstats +++ b/pyre/bench/synth/short_circuit_side_effects.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=7 +bridges_compiled=8 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1972 +guard_failures=2104 internal_compile_panics=0 loops_aborted=0 loops_compiled=4 +retraces_compiled=0 diff --git a/pyre/bench/synth/short_circuit_side_effects.wasm.jitstats b/pyre/bench/synth/short_circuit_side_effects.wasm.jitstats index ec62c587549..1a1da68a379 100644 --- a/pyre/bench/synth/short_circuit_side_effects.wasm.jitstats +++ b/pyre/bench/synth/short_circuit_side_effects.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=7 +bridges_compiled=8 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=1938 +guard_failures=2092 internal_compile_panics=0 loops_aborted=0 loops_compiled=4 diff --git a/pyre/bench/synth/str_search_index_bounds.cranelift.jitstats b/pyre/bench/synth/str_search_index_bounds.cranelift.jitstats index 803c1f3d1b6..7379af9dfb4 100644 --- a/pyre/bench/synth/str_search_index_bounds.cranelift.jitstats +++ b/pyre/bench/synth/str_search_index_bounds.cranelift.jitstats @@ -12,3 +12,4 @@ guard_failures=2094 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 +retraces_compiled=0 diff --git a/pyre/bench/synth/str_search_index_bounds.dynasm.jitstats b/pyre/bench/synth/str_search_index_bounds.dynasm.jitstats index 803c1f3d1b6..7379af9dfb4 100644 --- a/pyre/bench/synth/str_search_index_bounds.dynasm.jitstats +++ b/pyre/bench/synth/str_search_index_bounds.dynasm.jitstats @@ -12,3 +12,4 @@ guard_failures=2094 internal_compile_panics=0 loops_aborted=0 loops_compiled=5 +retraces_compiled=0 diff --git a/pyre/bench/synth/str_search_index_bounds.wasm.jitstats b/pyre/bench/synth/str_search_index_bounds.wasm.jitstats index ba0533346ae..1041d6848c0 100644 --- a/pyre/bench/synth/str_search_index_bounds.wasm.jitstats +++ b/pyre/bench/synth/str_search_index_bounds.wasm.jitstats @@ -12,3 +12,4 @@ guard_failures=2287 internal_compile_panics=0 loops_aborted=1 loops_compiled=5 +retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats index 88de6dd70fb..aa5441f3858 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.cranelift.jitstats @@ -2,7 +2,7 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=10 +fbw_blackhole_adopted_multi_frame=2 fbw_blackhole_adopted_single_frame=21 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 @@ -10,6 +10,6 @@ field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=31 -loops_compiled=2 +loops_aborted=23 +loops_compiled=50 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats index 88de6dd70fb..aa5441f3858 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.dynasm.jitstats @@ -2,7 +2,7 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=10 +fbw_blackhole_adopted_multi_frame=2 fbw_blackhole_adopted_single_frame=21 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 @@ -10,6 +10,6 @@ field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=31 -loops_compiled=2 +loops_aborted=23 +loops_compiled=50 retraces_compiled=0 diff --git a/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats b/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats index 6f5caa75783..003dfab98d3 100644 --- a/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats +++ b/pyre/bench/synth/trace_too_long_inline_multiframe.wasm.jitstats @@ -2,7 +2,7 @@ bridges_compiled=0 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 -fbw_blackhole_adopted_multi_frame=10 +fbw_blackhole_adopted_multi_frame=2 fbw_blackhole_adopted_single_frame=20 fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 @@ -10,6 +10,6 @@ field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 guard_failures=0 internal_compile_panics=0 -loops_aborted=30 -loops_compiled=0 +loops_aborted=22 +loops_compiled=48 retraces_compiled=0 diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats b/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats index db386c7fc38..ea0b79bc4c9 100644 --- a/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.cranelift.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=4 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=601 +guard_failures=1001 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats b/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats index db386c7fc38..ea0b79bc4c9 100644 --- a/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.dynasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=4 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,8 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=601 +guard_failures=1001 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 +retraces_compiled=0 diff --git a/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats b/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats index 1b618851f7b..ea0b79bc4c9 100644 --- a/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats +++ b/pyre/bench/synth/wasm_ca_trampoline_decline.wasm.jitstats @@ -1,4 +1,4 @@ -bridges_compiled=3 +bridges_compiled=4 descr_set_absent=0 descr_set_ambiguous=0 descr_set_stale_absent=0 @@ -8,7 +8,7 @@ fbw_rolled_back_with_effects=0 fbw_store_journal_rollback_failed=0 field_pos_attached_misplaced=0 field_pos_spec_misplaced=0 -guard_failures=601 +guard_failures=1001 internal_compile_panics=0 loops_aborted=1 loops_compiled=2 diff --git a/pyre/pyre-interpreter/src/baseobjspace.rs b/pyre/pyre-interpreter/src/baseobjspace.rs index cf5edf5db07..5574f8eb442 100644 --- a/pyre/pyre-interpreter/src/baseobjspace.rs +++ b/pyre/pyre-interpreter/src/baseobjspace.rs @@ -17849,12 +17849,10 @@ unsafe fn generator_invoke_execute_frame( if !ec.is_null() { (*ec).push_gen_or_coroutine(gen_obj); } - // generator.py:_invoke_execute_frame enters through the execution - // context of the thread that is resuming the generator. A suspended - // frame must not retain the context of the thread that created or last - // ran it: that thread may have exited before close()/finalization resumes - // the frame elsewhere. - frame.execution_context = ec; + // generator.py:_invoke_execute_frame uses the execution context of the + // thread resuming the generator. Like PyPy, the suspended frame stores no + // EC of its own; `execute_generator_frame` reads the thread-owned slot at + // this activation boundary. let result = frame.execute_generator_frame(w_inputvalue, operr, throw_args); let result = match result { Err(e) => { @@ -17885,7 +17883,6 @@ unsafe fn generator_invoke_execute_frame( }; // generator.py:142-145 `finally`. frame.f_backref = std::ptr::null_mut(); - frame.execution_context = std::ptr::null(); w_generator_set_running(gen_obj, false); if !ec.is_null() { (*ec).pop_gen_or_coroutine(gen_obj); diff --git a/pyre/pyre-interpreter/src/builtins.rs b/pyre/pyre-interpreter/src/builtins.rs index 159d3642652..0d5c8a904ef 100644 --- a/pyre/pyre-interpreter/src/builtins.rs +++ b/pyre/pyre-interpreter/src/builtins.rs @@ -12635,7 +12635,7 @@ fn builtin_compile(args: &[PyObjectRef]) -> Result if dont_inherit == 0 { let caller_frame = crate::eval::CURRENT_FRAME.with(|current| current.get()); if !caller_frame.is_null() { - let ec = unsafe { (*caller_frame).execution_context }; + let ec = crate::call::getexecutioncontext(); if !ec.is_null() { let top = unsafe { (*ec).gettopframe_nohidden() }; if !top.is_null() { @@ -13077,11 +13077,7 @@ fn exec_or_eval( // module is picked -- every one of which can collect. A frame a compiled // trace built moves there, so it is re-read out of the anchor at each. let caller_anchor = unsafe { crate::eval::FrameAnchor::from_raw(caller_frame) }; - let exec_ctx = if caller_frame.is_null() { - std::ptr::null::() - } else { - unsafe { (*caller_frame).execution_context } - }; + let exec_ctx = crate::call::getexecutioncontext(); // pyopcode.py ensure_ns — the globals object is the // user-supplied dict, else the caller frame's globals, else a fresh @@ -18996,14 +18992,7 @@ fn builtin_dunder_import(args: &[PyObjectRef]) -> Result() - } else { - unsafe { (*frame).execution_context } - } - }); + let exec_ctx = crate::call::getexecutioncontext(); // The native importer keys every lookup by `&str`, so a name that has no // such spelling goes straight to the app-level bootstrap. Re-read the // name through its root: `space_index_w` above may have moved it. diff --git a/pyre/pyre-interpreter/src/call.rs b/pyre/pyre-interpreter/src/call.rs index d21a3651b3b..725370725c6 100644 --- a/pyre/pyre-interpreter/src/call.rs +++ b/pyre/pyre-interpreter/src/call.rs @@ -521,15 +521,14 @@ pub(crate) fn capture_last_exec_ctx_cell() -> *const () { /// slot seeded at process boot by pyrex (`pyrex/src/lib.rs`'s /// `setup_exec_context`, which calls /// `set_last_exec_ctx(Rc::as_ptr(&execution_context))`) and -/// re-stamped on every `eval_frame_plain` entry. The slot stays -/// pointing at the root EC for the lifetime of the process, so +/// re-stamped on every `eval_frame_plain` entry. The slot stays +/// pointing at that thread's EC for the lifetime of its activation, so /// `sys.gettrace`/`settrace`/`getprofile`/`setprofile` and other /// `space.getexecutioncontext()` callers see the live EC even when /// no eval frame is currently on the stack. /// -/// TODO: pyre is single-threaded today so the TLS -/// slot is effectively a global. PyPy's per-thread `threadlocals` -/// dispatch lands when pyre adds its own thread state container. +/// The slot is per OS thread: `module::thread` installs the thread's own EC at +/// bootstrap and clears it at teardown, matching PyPy's threadlocals owner. pub fn getexecutioncontext() -> *const crate::PyExecutionContext { take_last_exec_ctx() } @@ -891,7 +890,7 @@ fn call_user_function_with_eval( args: &[PyObjectRef], eval_fn: EvalFn, ) -> PyResult { - let mut func_frame = match prepare_user_call(frame.execution_context, callable, args)? { + let mut func_frame = match prepare_user_call(getexecutioncontext(), callable, args)? { PreparedUserCall::Frame(func_frame) => func_frame, PreparedUserCall::Generator(generator) => return Ok(generator), }; @@ -1131,7 +1130,7 @@ enum CallMode { /// root is installed here and held across the whole dispatch. pub fn call_callable(frame: &mut PyFrame, callable: PyObjectRef, args: &[PyObjectRef]) -> PyResult { let _caller_locals_root = FrameLocalsRoot::new(frame); - call_callable_with_mode(frame.execution_context, callable, args, CallMode::Jit) + call_callable_with_mode(getexecutioncontext(), callable, args, CallMode::Jit) } /// `descroperation.py call_args(space, w_obj, args)` — the generic callable @@ -1230,7 +1229,7 @@ pub fn call_function_ex( ) -> PyResult { let _caller_locals_root = FrameLocalsRoot::new(frame); call_function_ex_in_ctx( - frame.execution_context, + getexecutioncontext(), callable, self_or_null, starargs, @@ -1353,7 +1352,7 @@ pub fn call_kw( ) -> PyResult { let _caller_locals_root = FrameLocalsRoot::new(frame); call_kw_in_ctx( - frame.execution_context, + getexecutioncontext(), callable, self_or_null, positional, @@ -1958,7 +1957,7 @@ pub fn call_callable_inline_residual( args: &[PyObjectRef], ) -> PyResult { let _caller_locals_root = FrameLocalsRoot::new(frame); - call_callable_with_mode(frame.execution_context, callable, args, CallMode::Plain) + call_callable_with_mode(getexecutioncontext(), callable, args, CallMode::Plain) } // ── __build_class__ implementation ─────────────────────────────────── @@ -2552,7 +2551,7 @@ pub fn call_with_kwargs( kwargs: &[(Wtf8Buf, PyObjectRef)], ) -> PyResult { let _caller_locals_root = FrameLocalsRoot::new(frame); - call_with_kwargs_in_ctx(frame.execution_context, callable, pos_args, kwargs) + call_with_kwargs_in_ctx(getexecutioncontext(), callable, pos_args, kwargs) } /// Call a user function with positional args + keyword args from a dict. diff --git a/pyre/pyre-interpreter/src/eval.rs b/pyre/pyre-interpreter/src/eval.rs index a971f2bddc6..8c5e388a0f3 100644 --- a/pyre/pyre-interpreter/src/eval.rs +++ b/pyre/pyre-interpreter/src/eval.rs @@ -149,7 +149,7 @@ pub fn install_current_frame(frame: &mut PyFrame) -> CurrentFrameGuard { // can iterate all active frames. `eval_frame_plain` calls // `ExecutionContext::enter` before installing TLS-only state, but // the JIT portal path enters through this helper directly. - let ec = frame.execution_context as *mut PyExecutionContext; + let ec = crate::call::getexecutioncontext() as *mut PyExecutionContext; let previous_ec_top = if ec.is_null() { std::ptr::null_mut() } else { @@ -918,20 +918,9 @@ pub unsafe fn walk_pyframe_roots_area( // and are forwarded by the GC's root walker; no extra visit here. let mut frame = cf.get(); - let frame_ec = if frame.is_null() { - std::ptr::null_mut() - } else { - unsafe { (*frame).execution_context as *mut PyExecutionContext } - }; - // Root the EC slots from the current frame's EC AND the ambient - // TLS EC (`getexecutioncontext`). The ambient visit covers the - // spans where no frame is installed in `CURRENT_FRAME` yet the EC - // is live — between `ExecutionContext::enter` and `eval_loop`'s - // frame install, and around `return_trace`/`leave` after the - // frame guard drops — where `sys_exc_value` may already hold a - // nursery exception. PyPy reaches the ExecutionContext - // unconditionally through `space.threadlocals`, independent of - // any frame. + // Root the EC slots from the ambient TLS EC. PyPy reaches the + // ExecutionContext unconditionally through `space.threadlocals`, + // independent of whether a frame is currently installed. let ambient_ec = unsafe { (&*(area.last_exec_ctx as *const Cell<*const PyExecutionContext>)).get() as *mut PyExecutionContext @@ -974,10 +963,7 @@ pub unsafe fn walk_pyframe_roots_area( visitor(unsafe { &mut *(hook as *mut majit_ir::GcRef) }); } }; - visit_ec_slots(frame_ec); - if ambient_ec != frame_ec { - visit_ec_slots(ambient_ec); - } + visit_ec_slots(ambient_ec); while !frame.is_null() { // SAFETY: PyFrame pointers on the f_backref chain are valid // for the duration of the enclosing `eval_with_jit` call. A @@ -1066,11 +1052,11 @@ pub unsafe fn walk_pyframe_roots_area( as *mut PyObjectRef; visitor(&mut *(w_dict_slot as *mut majit_ir::GcRef)); } - // pyframe.py `self.w_globals` is the dict OBJECT. Forward - // the field before following the dict's own storage. - let w_globals_obj_slot = &mut (*frame).w_globals as *mut PyObjectRef; - visitor(&mut *(w_globals_obj_slot as *mut majit_ir::GcRef)); - // pyframe.py `debugdata.w_locals` (the frame's locals + // pyframe.py `FrameDebugData.w_globals` / `w_locals`: only a + // code/globals identity mismatch stores globals on the frame. + // The common globals edge is reached through `pycode` above. + // Both debug mappings must be forwarded before use. + // `debugdata.w_locals` (the frame's locals // mapping object) and `w_f_trace` carry GCREFs that survive // the frame; forward both slots. The locals mapping holds its // own bindings (module globals, class namespace, function @@ -1083,6 +1069,8 @@ pub unsafe fn walk_pyframe_roots_area( visitor(&mut *(debugdata_slot as *mut majit_ir::GcRef)); } let d = &mut *(*frame).debugdata; + let w_globals_slot = &mut d.w_globals as *mut PyObjectRef; + visitor(&mut *(w_globals_slot as *mut majit_ir::GcRef)); let w_locals_slot = &mut d.w_locals as *mut PyObjectRef; visitor(&mut *(w_locals_slot as *mut majit_ir::GcRef)); let w_extra_locals_slot = &mut d.w_extra_locals as *mut PyObjectRef; @@ -1099,7 +1087,7 @@ pub unsafe fn walk_pyframe_roots_area( &mut (*frame).lastblock as *mut *mut crate::pyframe::FrameBlock; visitor(&mut *(lastblock_slot as *mut majit_ir::GcRef)); } - let live_obj = (*frame).w_globals; + let live_obj = (*frame).get_w_globals(); // For a W_ModuleDictObject the LOAD_GLOBAL read path consults the // authoritative `dstorage` cell map / `object_storage` / // strategy caches. Forward those movable @@ -1419,12 +1407,8 @@ pub fn walk_suspended_generator_frame( let yielding_slot = &mut (*frame).w_yielding_from as *mut PyObjectRef; visitor(&mut *(yielding_slot as *mut majit_ir::GcRef)); - // Forward the globals/builtin object pointers; their dict VALUES are - // not walked here — a module dict is rooted globally by - // `walk_module_dicts_gc`, and a GC-managed `exec` globals dict is - // reached transitively through its own trace. - let w_globals_obj_slot = &mut (*frame).w_globals as *mut PyObjectRef; - visitor(&mut *(w_globals_obj_slot as *mut majit_ir::GcRef)); + // Forward the builtin object pointer. The common globals object is + // reached through `pycode`; the rare override is in debugdata below. let w_builtin_slot = &mut (*frame).w_builtin as *mut PyObjectRef; visitor(&mut *(w_builtin_slot as *mut majit_ir::GcRef)); let w_builtin = (*frame).w_builtin; @@ -1441,6 +1425,8 @@ pub fn walk_suspended_generator_frame( visitor(&mut *(debugdata_slot as *mut majit_ir::GcRef)); } let d = &mut *(*frame).debugdata; + let w_globals_slot = &mut d.w_globals as *mut PyObjectRef; + visitor(&mut *(w_globals_slot as *mut majit_ir::GcRef)); let w_locals_slot = &mut d.w_locals as *mut PyObjectRef; visitor(&mut *(w_locals_slot as *mut majit_ir::GcRef)); let w_extra_locals_slot = &mut d.w_extra_locals as *mut PyObjectRef; @@ -1820,7 +1806,7 @@ pub fn handle_exception_with_context( // which routes through the `attach_tb=False` branch, so all three // tracing hooks are skipped per `:91-94`. Pyre carries the same // intent via `PyError.attach_tb` set by `eval.rs::reraise`. - let ec = frame.execution_context as *mut crate::PyExecutionContext; + let ec = crate::call::getexecutioncontext() as *mut crate::PyExecutionContext; // Everything below allocates before it touches the frame again: // `to_exc_object` materialises the exception, `chain_context` builds the // `__context__` link, the trace hooks run arbitrary Python and @@ -2134,14 +2120,14 @@ pub(crate) fn eval_frame_plain_with_resume( operr, throw_args, }; - if frame.execution_context.is_null() { + let ec = crate::call::getexecutioncontext() as *mut crate::PyExecutionContext; + if ec.is_null() { if let Some(value) = prepare_frame_resume_for_dispatch(frame, &mut resume)? { return Ok(value); } - return eval_loop(frame); + return eval_loop(frame, ec); } - let execution_context = - unsafe { &mut *(frame.execution_context as *mut crate::PyExecutionContext) }; + let execution_context = unsafe { &mut *ec }; // executioncontext.py / threadlocals.py parity: the current // ExecutionContext is owned by the OS-thread locals and is installed by // thread bootstrap. Entering an (including inlined) frame must not @@ -2174,7 +2160,7 @@ pub(crate) fn eval_frame_plain_with_resume( w_exitvalue = value; return Ok(value); } - let result = eval_loop(frame)?; + let result = eval_loop(frame, ec)?; w_exitvalue = result; Ok(result) })(); @@ -2207,10 +2193,11 @@ pub(crate) fn eval_frame_plain_with_resume( /// Resume interpretation after compiled code guard failure. pub fn eval_loop_for_force(frame: &mut PyFrame) -> PyResult { - eval_loop(frame) + let ec = crate::call::getexecutioncontext() as *mut crate::PyExecutionContext; + eval_loop(frame, ec) } -fn eval_loop(frame: &mut PyFrame) -> PyResult { +fn eval_loop(frame: &mut PyFrame, ec: *mut crate::PyExecutionContext) -> PyResult { // Bump the monotonic frame eval-loop entry odometer: a user Python frame // is about to run bytecode. The FBW FOR_ITER Option-C guard snapshots // this around a residual call to detect a body effect that ran through @@ -2227,7 +2214,7 @@ fn eval_loop(frame: &mut PyFrame) -> PyResult { // this bit out. majit_ir::eval_breaker_word::set_gc_interp(); } - let _current_frame_guard = if frame.execution_context.is_null() { + let _current_frame_guard = if ec.is_null() { install_current_frame(frame) } else { install_current_frame_tls_only(frame) @@ -2282,7 +2269,6 @@ fn eval_loop(frame: &mut PyFrame) -> PyResult { // — bytecode_trace fires bytecode_only_trace then decrements // the ticker. Gated upstream on `w_tracefunc.is_null()` so the // no-tracer hot path is a single null-check + ticker decrement. - let ec = frame.execution_context as *mut crate::PyExecutionContext; if !ec.is_null() { if frame.take_failed_attr_before_opcode() { unsafe { (*ec).run_failed_attr_finalizers() }; @@ -3970,7 +3956,7 @@ impl OpcodeStepExecutor for PyFrame { self.get_w_globals(), pyre_object::w_none(), 0, - self.execution_context, + crate::call::getexecutioncontext(), ) } @@ -4023,7 +4009,7 @@ impl OpcodeStepExecutor for PyFrame { // Stack: [module] → peek module, push getattr(module, name) fn import_from(&mut self, name: &str) -> Result<(), PyError> { let module = self.peek(); - let ec = self.execution_context; + let ec = crate::call::getexecutioncontext(); let anchor = FrameAnchor::new(self); let attr = crate::importing::import_from(module, name, ec)?; Self::push_anchored(&anchor, attr) @@ -5537,7 +5523,7 @@ except AttributeError as exc: let (res, frame) = run_exec_frame(source); res.expect("member slot AttributeError regression"); unsafe { - let value = w_dict_getitem_str(frame.w_globals, "result").unwrap(); + let value = w_dict_getitem_str(frame.get_w_globals(), "result").unwrap(); assert_eq!( w_str_get_wtf8(value).as_str(), Ok("'__main__.make_type..X' object has no attribute 'a'") @@ -5653,7 +5639,7 @@ result = ( let (res, frame) = run_exec_frame(source); res.expect("CPython list allocation metadata failed"); unsafe { - let result = w_dict_getitem_str(frame.w_globals, "result").unwrap(); + let result = w_dict_getitem_str(frame.get_w_globals(), "result").unwrap(); assert!(crate::baseobjspace::is_true(result).unwrap()); } } diff --git a/pyre/pyre-interpreter/src/function.rs b/pyre/pyre-interpreter/src/function.rs index 806cc04c033..32c83992744 100644 --- a/pyre/pyre-interpreter/src/function.rs +++ b/pyre/pyre-interpreter/src/function.rs @@ -3775,7 +3775,7 @@ fn _flat_pycall( code, &[], // locals filled below directly from stack w_globals, - frame.execution_context, + crate::call::getexecutioncontext(), closure, crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { @@ -3854,7 +3854,7 @@ fn _flat_pycall_defaults( code, &[], // locals filled below w_globals, - frame.execution_context, + crate::call::getexecutioncontext(), closure, crate::pyframe::FrameLocalsArrayAllocation::OldGenGc, ) { diff --git a/pyre/pyre-interpreter/src/module/__pypy__/interp_buffer.rs b/pyre/pyre-interpreter/src/module/__pypy__/interp_buffer.rs index d3339437388..968d3324ece 100644 --- a/pyre/pyre-interpreter/src/module/__pypy__/interp_buffer.rs +++ b/pyre/pyre-interpreter/src/module/__pypy__/interp_buffer.rs @@ -577,11 +577,7 @@ pub(crate) fn is_contiguous(obj: PyObjectRef) -> Result { /// The `memoryview` builtin type via the live execution context. fn memoryview_type() -> Option { - let frame = crate::eval::current_frame(); - if frame.is_null() { - return None; - } - let ec = unsafe { (*frame).execution_context }; + let ec = crate::call::getexecutioncontext(); if ec.is_null() { return None; } diff --git a/pyre/pyre-interpreter/src/module/_pickle/mod.rs b/pyre/pyre-interpreter/src/module/_pickle/mod.rs index b3cbe549fb6..c3102523d59 100644 --- a/pyre/pyre-interpreter/src/module/_pickle/mod.rs +++ b/pyre/pyre-interpreter/src/module/_pickle/mod.rs @@ -221,14 +221,9 @@ pub(crate) fn import_module(name: &str) -> Result { .ok_or_else(|| PyError::value_error(format!("Can't find module {name:?} in sys.modules"))) } -/// The live execution context reached via the current frame, or `None` -/// when no frame is on the stack. +/// The live execution context reached through the thread-owned space state. fn current_ec() -> Option<*const crate::PyExecutionContext> { - let frame = crate::eval::current_frame(); - if frame.is_null() { - return None; - } - let ec = unsafe { (*frame).execution_context }; + let ec = crate::call::getexecutioncontext(); if ec.is_null() { None } else { Some(ec) } } diff --git a/pyre/pyre-interpreter/src/module/_pickle/unpickler.rs b/pyre/pyre-interpreter/src/module/_pickle/unpickler.rs index b352bd91d77..55d28af9995 100644 --- a/pyre/pyre-interpreter/src/module/_pickle/unpickler.rs +++ b/pyre/pyre-interpreter/src/module/_pickle/unpickler.rs @@ -763,12 +763,7 @@ fn load_readonly_buffer(slot: usize) -> Result<(), PyError> { /// The `memoryview` builtin type via the live execution context. fn memoryview_type() -> Result { - let frame = crate::eval::CURRENT_FRAME.with(|f| f.get()); - let ec = if frame.is_null() { - std::ptr::null() - } else { - unsafe { (*frame).execution_context } - }; + let ec = crate::call::getexecutioncontext(); if ec.is_null() { return Err(unpickling_error("memoryview type unavailable")); } diff --git a/pyre/pyre-interpreter/src/module/sys/vm.rs b/pyre/pyre-interpreter/src/module/sys/vm.rs index 48f020bd517..f295faf0ecf 100644 --- a/pyre/pyre-interpreter/src/module/sys/vm.rs +++ b/pyre/pyre-interpreter/src/module/sys/vm.rs @@ -1600,7 +1600,7 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { if current.is_null() { return Ok(pyre_object::w_none()); } - // `w_globals` is one of the six fields `interp_jit.py` + // `get_w_globals` reads `pycode`, which `interp_jit.py` // declares virtualizable, so the frame it is read off has to be // materialized first. The force belongs HERE, at the consumer, and // not at the walk that reached the frame — see [`force_frame`]: @@ -1608,7 +1608,7 @@ pub fn register_module(ns: pyre_object::PyObjectRef) { // `vable_after_residual_call` aborts the trace with ABORT_ESCAPE. let anchor = unsafe { crate::eval::FrameAnchor::from_raw(current) }; crate::executioncontext::force_frame(current); - let w_globals = unsafe { (*anchor.live()).w_globals }; + let w_globals = unsafe { (*anchor.live()).get_w_globals() }; if w_globals.is_null() { return Ok(pyre_object::w_none()); } diff --git a/pyre/pyre-interpreter/src/pycode.rs b/pyre/pyre-interpreter/src/pycode.rs index 3bf04e4ba55..27cc3bf5792 100644 --- a/pyre/pyre-interpreter/src/pycode.rs +++ b/pyre/pyre-interpreter/src/pycode.rs @@ -2695,7 +2695,13 @@ pub unsafe fn w_code_get_w_globals(obj: PyObjectRef) -> PyObjectRef { if obj.is_null() { return pyre_object::PY_NULL; } - unsafe { (*(obj as *const PyCode)).w_globals } + // Paired with the publication in `w_code_frame_stores_global`. + unsafe { + std::sync::atomic::AtomicPtr::from_ptr(std::ptr::addr_of_mut!( + (*(obj as *mut PyCode)).w_globals + )) + .load(std::sync::atomic::Ordering::Acquire) + } } /// PyPy: `PyCode.w_globals = w_globals`. @@ -2708,7 +2714,10 @@ pub unsafe fn w_code_set_w_globals(obj: PyObjectRef, w_globals: PyObjectRef) { return; } unsafe { - (*(obj as *mut PyCode)).w_globals = w_globals; + std::sync::atomic::AtomicPtr::from_ptr(std::ptr::addr_of_mut!( + (*(obj as *mut PyCode)).w_globals + )) + .store(w_globals, std::sync::atomic::Ordering::Release); } // A bootstrap code slot is reached only by the prebuilt root walk, which // clean minor collections may skip; record the store. @@ -2729,16 +2738,36 @@ pub unsafe fn w_code_frame_stores_global(obj: PyObjectRef, w_globals: PyObjectRe if obj.is_null() { return false; } - let code = unsafe { &mut *(obj as *mut PyCode) }; - if code.w_globals.is_null() { - code.w_globals = w_globals; - // Prebuilt-family store (see `w_code_set_w_globals`). - publish_code_slot_store(obj); - register_live_code_wrapper(code.code_ptr, obj); - register_w_globals_stamped_code(obj); - return false; + // `pycode.py frame_stores_global` reads the slot and stores into it under + // the GIL, which makes the pair indivisible. Pyre is free-threaded, and a + // `false` answer is what makes a frame take its globals from the code + // object: two threads first running one code object in different globals + // both read the null, and an unsynchronized store would let both answer + // `false`, leaving the loser's frame reading the winner's namespace. + // Publish by compare-exchange and answer against whichever pointer won. + let code = obj as *mut PyCode; + let slot = unsafe { + std::sync::atomic::AtomicPtr::from_ptr(std::ptr::addr_of_mut!((*code).w_globals)) + }; + let mut published = slot.load(std::sync::atomic::Ordering::Acquire); + if published.is_null() { + match slot.compare_exchange( + pyre_object::PY_NULL, + w_globals, + std::sync::atomic::Ordering::AcqRel, + std::sync::atomic::Ordering::Acquire, + ) { + Ok(_) => { + // Prebuilt-family store (see `w_code_set_w_globals`). + publish_code_slot_store(obj); + register_live_code_wrapper(unsafe { (*code).code_ptr }, obj); + register_w_globals_stamped_code(obj); + return false; + } + Err(winner) => published = winner, + } } - !std::ptr::eq(code.w_globals, w_globals) + !std::ptr::eq(published, w_globals) } /// The state of a code object's `co_positions` rows once its own `locations` diff --git a/pyre/pyre-interpreter/src/pyframe.rs b/pyre/pyre-interpreter/src/pyframe.rs index 614e4404a5e..8d015502476 100644 --- a/pyre/pyre-interpreter/src/pyframe.rs +++ b/pyre/pyre-interpreter/src/pyframe.rs @@ -659,10 +659,6 @@ pub struct PyFrame { /// other `PyObject`-layout struct so `ob_type` reads land on the /// typeptr the JIT `GuardClass` / `type()` expect. pub ob_header: PyObject, - /// Raw pointer to the shared execution context. - /// The top-level frame leaks the Rc via `Rc::into_raw`. - /// Callee frames just copy the pointer (no atomic refcount ops). - pub execution_context: *const PyExecutionContext, /// Pointer to the Code object (PyCode). /// /// PyPy: pyframe.py `self.pycode = code` — stores the PyCode instance. @@ -708,11 +704,13 @@ pub struct PyFrame { /// receiver consumed by a failing `LOAD_ATTR`. PyPy's tracing GC does /// not need this state, but pyre must retain it on the owning red frame /// until the exception stack has discarded the receiver. It is a - /// separate ordinary frame word rather than extra bits in `flags`. PyPy - /// leaves frame state that is not named by `_virtualizable_` on the - /// concrete red frame; traced accesses therefore remain ordinary - /// getfield/setfield operations on this frame's identity. - pub failed_attr_cleanup: usize, + /// separate ordinary byte beside `flags`. PyPy represents the adjacent + /// frame-status booleans as byte fields; this CPython-only state has only + /// six values, so widening it to a machine word would make every hot call + /// frame pay for cold exception cleanup. It remains a concrete red-frame + /// getfield/setfield, just with the narrow storage PyPy's rtyper would + /// select for the same finite state. + pub failed_attr_cleanup: u8, /// pyframe.py:82 debugdata — lazily allocated tracing/debug payload. /// Virtualizable static field (interp_jit.py:28). pub debugdata: *mut FrameDebugData, @@ -743,16 +741,6 @@ pub struct PyFrame { /// `pyopcode.py:773-774`) see the picked Module, not the EC's /// default builtin. pub w_builtin: PyObjectRef, - /// `pypy/interpreter/pyframe.py:49 self.w_globals = w_globals` - /// — the canonical W_DictObject paired with this frame's globals. - /// - /// **Population**: threaded through directly by the object-taking - /// constructors; storage-only builders use `PY_NULL`. Every reader after - /// construction observes the same value across the frame's lifetime. This - /// is the source of truth: `get_w_globals()` and - /// `get_w_globals_storage()` both return it directly. Synthetic test stubs - /// that hand-build PyFrame without a real globals leave it `PY_NULL`. - pub w_globals: PyObjectRef, } /// GC type id for `PyFrame`. Reserved ahead of any callsite that allocates @@ -1015,7 +1003,6 @@ impl FrameBox { frame.w_yielding_from, frame.f_backref as pyre_object::PyObjectRef, frame.w_builtin, - frame.w_globals, ]); let raw = pyre_object::gc_hook::try_gc_alloc_stable_raw( PYFRAME_GC_TYPE_ID, @@ -1041,7 +1028,6 @@ impl FrameBox { frame.w_yielding_from = frame_root.get(inputs + 6); frame.f_backref = frame_root.get(inputs + 7) as *mut PyFrame; frame.w_builtin = frame_root.get(inputs + 8); - frame.w_globals = frame_root.get(inputs + 9); debug_assert!(pyre_object::gc_hook::try_gc_owns_object( frame.locals_cells_stack_w as *mut u8 )); @@ -1197,7 +1183,7 @@ impl FrameBox { slot }); let coroutine_origin_slot = if is_coroutine { - let origin = capture_coroutine_origin(self.execution_context); + let origin = capture_coroutine_origin(crate::call::getexecutioncontext()); pyre_object::gc_roots::pin_root(origin); Some(pyre_object::gc_roots::shadow_stack_len() - 1) } else { @@ -1702,7 +1688,9 @@ pub fn report_stack_underflow(frame: &PyFrame) { /// `call_trace` / `bytecode_trace` are driven from the plain eval path. /// A frame carrying its own `f_trace` also runs interpreted. pub fn frame_tracing_active(frame: &PyFrame) -> bool { - let ec = frame.execution_context; + // PyPy's `PyFrame.get_is_being_profiled` consults the thread-owned + // ExecutionContext through `self.space`; it is not frame storage. + let ec = crate::call::getexecutioncontext(); if ec.is_null() { return false; } @@ -1713,6 +1701,11 @@ pub fn frame_tracing_active(frame: &PyFrame) -> bool { #[repr(C)] #[derive(Clone)] pub struct FrameDebugData { + /// `FrameDebugData.__init__` / `PyFrame.get_w_globals`: the code object's + /// first globals lives on `PyCode`; only a frame running that same code in + /// another globals object stores the override here. This is the one-field + /// frame-size optimization introduced by PyPy commit `bcd8653e5ec`. + pub w_globals: PyObjectRef, /// pyframe.py:44 — the frame's locals mapping (`self.w_locals`). /// At module scope it is the `w_globals` dict; in a class body it is /// the class namespace; for a function it is the dict lazily @@ -1746,12 +1739,15 @@ pub struct FrameDebugData { } impl FrameDebugData { - // `_pycode` keeps the constructor shape of `pyframe.py:48 - // FrameDebugData.__init__(self, pycode, init_lineno)`. The frame's - // globals object now lives in `PyFrame.w_globals`, so debugdata no - // longer snapshots `pycode.w_globals`. - pub fn new(_pycode: *const (), init_lineno: isize) -> Self { + /// `pyframe.py FrameDebugData.__init__`: initialize the rare-frame + /// globals override from the code object's first-seen globals. + pub fn new(pycode: *const (), init_lineno: isize) -> Self { Self { + w_globals: if pycode.is_null() { + pyre_object::PY_NULL + } else { + unsafe { crate::w_code_get_w_globals(pycode as PyObjectRef) } + }, w_locals: pyre_object::PY_NULL, w_extra_locals: pyre_object::PY_NULL, w_f_trace: pyre_object::PY_NULL, @@ -1775,6 +1771,10 @@ impl Default for FrameDebugData { /// Byte offset of `w_locals` in `FrameDebugData`. pub const FRAME_DEBUG_DATA_W_LOCALS_OFFSET: usize = std::mem::offset_of!(FrameDebugData, w_locals); +/// Byte offset of the rare per-frame globals override. +pub const FRAME_DEBUG_DATA_W_GLOBALS_OFFSET: usize = + std::mem::offset_of!(FrameDebugData, w_globals); + /// Allocated size of a `FrameDebugData`. pub const FRAME_DEBUG_DATA_SIZE: usize = std::mem::size_of::(); @@ -1886,13 +1886,6 @@ pub const PYFRAME_F_BACKREF_OFFSET: usize = std::mem::offset_of!(PyFrame, f_back /// guard exit / re-entry edge. pub const PYFRAME_W_BUILTIN_OFFSET: usize = std::mem::offset_of!(PyFrame, w_builtin); -/// Byte offset of `w_globals` in `PyFrame` — the canonical -/// W_DictObject paired with the storage in `w_globals`. Registered -/// as a GC-traceable slot so a minor collection forwards the pointer -/// when the dict survives. The slot is lazy: `PY_NULL` until -/// `get_w_globals` resolves it. -pub const PYFRAME_W_GLOBALS_OFFSET: usize = std::mem::offset_of!(PyFrame, w_globals); - // Backward-compat aliases used by JIT code. pub const PYFRAME_STACK_DEPTH_OFFSET: usize = PYFRAME_VALUESTACKDEPTH_OFFSET; pub const PYFRAME_LOCALS_OFFSET: usize = PYFRAME_LOCALS_CELLS_STACK_OFFSET; @@ -2577,26 +2570,40 @@ impl PyFrame { self.code() } - /// pyframe.py:129-133 get_w_globals_storage — return the frame's canonical - /// W_DictObject. `pyframe.py:49 self.w_globals = w_globals` keeps that - /// object as the single globals field. + /// PyPy `get_w_globals_storage` compatibility alias. #[inline] pub fn get_w_globals_storage(&self) -> PyObjectRef { - self.w_globals + self.get_w_globals() } - /// The canonical W_DictObject for this frame's globals - /// (`pyframe.py:49 self.w_globals = w_globals`). Every frame - /// constructor seeds `w_globals` eagerly, so this is a plain - /// field read; callers wanting object identity - /// (`function.__globals__ is frame.f_globals`, `globals() is - /// module.__dict__`, etc.) read it directly. - /// - /// Returns `PY_NULL` when the frame has no globals (test stubs); - /// callers that expect a dict should null-check before dereferencing. + /// `pyframe.py PyFrame.get_w_globals`: the normal case reads the + /// promoted code object's first-seen globals. A frame executing a shared + /// code object in another namespace carries the override in debugdata. #[inline] pub fn get_w_globals(&self) -> PyObjectRef { - self.w_globals + if let Some(data) = self.getdebug_data() { + return data.w_globals; + } + if self.pycode.is_null() { + return pyre_object::PY_NULL; + } + // `pyframe.py get_w_globals` `return jit.promote(self.pycode).w_globals`. Without + // the promote the code object stays a varying value on the trace and + // the globals read is a load the optimizer cannot fold; `w_globals` is + // not a virtualizable field, so nothing else pins it to a constant. + let pycode = majit_metainterp::jit::promote(self.pycode); + unsafe { crate::w_code_get_w_globals(pycode as PyObjectRef) } + } + + /// `pyframe.py PyFrame.set_w_globals`: force the rare globals override + /// into lazily-created debugdata. Root the value across that allocation; + /// RPython's shadow-stack transform does the same for the live argument. + pub fn set_w_globals(&mut self, w_globals: PyObjectRef) { + let roots = pyre_object::gc_roots::push_roots(); + let slot = roots.base(); + roots.pin_root(w_globals); + self.getorcreate_debug_data(-1).w_globals = roots.get(slot); + remember_frame_debug_data(self.debugdata); } /// pyframe.py get_w_f_trace @@ -2669,21 +2676,14 @@ impl PyFrame { clear_debugdata_ptr(&mut self.debugdata); clear_block_chain(&mut self.lastblock); } - // `pyframe.py:114-115` — `self.builtin = space.builtin.pick_builtin( - // w_globals)`. pyre keeps the picked builtin and the canonical - // `w_globals` W_DictObject (the `get_w_globals_storage` resolution of - // `pyframe.py:128-132`) in the adjacent `w_builtin` / `w_globals` - // slots. This storage-only hook carries no globals object, so a - // frame built through this hook is left in the same state as one - // built by `createframe`. + // This storage-only hook carries no globals object. let w_globals = PY_NULL; - self.w_builtin = crate::baseobjspace::frame_builtin_obj(w_globals, self.execution_context); - self.w_globals = w_globals; - // pyframe.py — stamp `pycode.w_globals` (the first-globals cache - // the LOAD_GLOBAL fast path keys on); side effect only, since the - // gated debugdata snapshot retired in favour of `w_globals`. - unsafe { - crate::w_code_frame_stores_global(code as PyObjectRef, self.w_globals); + self.w_builtin = + crate::baseobjspace::frame_builtin_obj(w_globals, crate::call::getexecutioncontext()); + // `pyframe.py PyFrame.__init__`: only a code/globals identity mismatch + // creates a per-frame debugdata override. + if unsafe { crate::w_code_frame_stores_global(code as PyObjectRef, w_globals) } { + self.set_w_globals(w_globals); } // pyframe.py — final step of __init__. self.initialize_frame_scopes(outer_func, code).expect( @@ -2971,16 +2971,19 @@ impl PyFrame { /// `createframe` (PyPy `baseobjspace.py`) so every heap-allocated /// `PyFrame` flows through the canonical entry point. pub fn new(code: CodeObject) -> FrameBox { - let frame = Self::new_with_context(code, Rc::new(PyExecutionContext::default())) + let execution_context = Rc::new(PyExecutionContext::default()); + let ctx_ptr = Rc::as_ptr(&execution_context); + let frame = Self::new_with_context(code, execution_context) .expect("PyFrame::new: test entry code must not carry freevars"); // `threadlocals.py:enter_thread` — the ExecutionContext slot belongs to // the OS-thread locals and is installed once at thread entry, not // re-stamped per frame. The other entry points do this themselves - // (`pyrex/src/lib.rs`, `pyre-wasm/src/lib.rs`); as the remaining entry - // point this one must too, or `space.getexecutioncontext()` stays null - // and every context reached through it — the class-body frame's - // builtins among them — falls back to the empty default. - crate::call::set_last_exec_ctx(frame.execution_context); + // (`pyrex/src/lib.rs`, `pyre-wasm/src/lib.rs`, + // `pyre-wasm-test/src/main.rs`); as the remaining entry point this one + // must too, or `space.getexecutioncontext()` stays null and every + // context reached through it — the class-body frame's builtins among + // them — falls back to the empty default. + crate::call::set_last_exec_ctx(ctx_ptr); frame } @@ -3086,14 +3089,13 @@ impl PyFrame { /// the generator snapshot. Frame-LOCAL state (`locals_cells_stack_w` / /// `valuestackdepth` / `last_instr`) is COPIED, so snapshot mutations to /// locals/stack are discarded — the abort-safety the snapshot exists - /// for. `w_globals` is the SAME dict ptr, so a concrete shared-heap + /// for. The code/debugdata-derived globals is the SAME dict ptr, so a concrete shared-heap /// write during recording would leak to the real heap and double-apply /// on the compiled loop's re-run; Gap 10 removed that path (inline-frame /// STORE_GLOBAL records as deferred IR, applied exactly once). fn build_snapshot_frame(&self, allocation: FrameLocalsArrayAllocation) -> PyFrame { PyFrame { ob_header: frame_ob_header(), - execution_context: self.execution_context, pycode: self.pycode, locals_cells_stack_w: unsafe { let values = locals_w!(self).to_vec(); @@ -3115,7 +3117,6 @@ impl PyFrame { w_yielding_from: self.w_yielding_from, f_backref: self.f_backref, w_builtin: self.w_builtin, - w_globals: self.w_globals, } } @@ -3706,9 +3707,9 @@ impl PyFrame { /// `frame_finished_execution` status bit. pub const FLAG_FRAME_FINISHED: u8 = 0b10; /// A failed LOAD_ATTR is waiting for this frame's POP_EXCEPT cleanup. - const FAILED_ATTR_AFTER_POP_EXCEPT: usize = 4; + const FAILED_ATTR_AFTER_POP_EXCEPT: u8 = 4; /// The next dispatch boundary must collect the failed LOAD_ATTR receiver. - const FAILED_ATTR_BEFORE_OPCODE: usize = usize::MAX; + const FAILED_ATTR_BEFORE_OPCODE: u8 = u8::MAX; /// pyframe.py:80 `escaped`. #[inline] @@ -3796,10 +3797,11 @@ impl PyFrame { if !self.w_builtin.is_null() { return self.w_builtin; } - if self.execution_context.is_null() { + let execution_context = crate::call::getexecutioncontext(); + if execution_context.is_null() { return pyre_object::PY_NULL; } - unsafe { (*self.execution_context).get_builtin() } + unsafe { (*execution_context).get_builtin() } } /// `frame.f_backref()` — force the caller vref. `f_backref` holds a @@ -4769,15 +4771,12 @@ impl PyFrame { // remember the completed array before the next allocating operation. remember_frame_locals_array(locals_cells_stack_w); - // pyframe.py — stamp `pycode.w_globals`; side effect only (the - // gated debugdata snapshot retired in favour of `w_globals`). - unsafe { - crate::w_code_frame_stores_global(_roots.get(root_base), _roots.get(root_base + 1)); - } + let frame_stores_global = unsafe { + crate::w_code_frame_stores_global(_roots.get(root_base), _roots.get(root_base + 1)) + }; let mut frame = PyFrame { ob_header: frame_ob_header(), - execution_context, pycode: _roots.get(root_base) as *const (), locals_cells_stack_w, valuestackdepth: num_locals + num_cells, @@ -4791,8 +4790,10 @@ impl PyFrame { w_yielding_from: PY_NULL, f_backref: std::ptr::null_mut(), w_builtin: _roots.get(root_base + 3), - w_globals: _roots.get(root_base + 1), }; + if frame_stores_global { + frame.set_w_globals(_roots.get(root_base + 1)); + } // This constructor bypasses `initialize_frame_scopes`, so apply the // scope binding it would have done. `FunctionType(co, globals)` over // a module-level code object arrives here, and without the binding its @@ -5044,7 +5045,7 @@ pub fn createframe( } /// `baseobjspace.py createframe` with the globals passed as the dict -/// OBJECT (`pyframe.py:49 self.w_globals = w_globals` stores the object). +/// object; `PyCode.frame_stores_global` chooses code storage or debugdata. pub fn createframe_obj( code: *const (), w_globals: PyObjectRef, @@ -5064,26 +5065,20 @@ pub fn createframe_obj( // ... // self.initialize_frame_scopes(outer_func, code) // - // `pyframe.py:49 self.w_globals = w_globals`: preserve the exact object. - // This is observable for dict subclasses (notably annotationlib's - // `_StringifierDict`, whose `__missing__` creates ForwardRef values) and - // is the identity MAKE_FUNCTION must pass on to each callee frame. + // Preserve the exact object either on `PyCode.w_globals` (normal case) or + // the rare `FrameDebugData.w_globals` override. let raw = unsafe { crate::w_code_get_ptr(code as PyObjectRef) as *const CodeObject }; let code_ref = unsafe { &*raw }; let num_locals = code_ref.varnames.len(); let num_cells = ncells(code_ref); let max_stack = code_ref.max_stackdepth as usize; - // pyframe.py — stamp `pycode.w_globals`; side effect only (the gated - // debugdata snapshot retired in favour of `w_globals`). - unsafe { - crate::w_code_frame_stores_global(code as PyObjectRef, w_globals); - } + let frame_stores_global = + unsafe { crate::w_code_frame_stores_global(code as PyObjectRef, w_globals) }; let size = num_locals + num_cells + max_stack; let w_builtin = crate::baseobjspace::frame_builtin_obj(w_globals, execution_context); let mut frame = FrameBox::new(PyFrame { ob_header: frame_ob_header(), - execution_context, pycode: code, locals_cells_stack_w: unsafe { alloc_frame_locals_array(size, PY_NULL, FrameLocalsArrayAllocation::OldGenGc) @@ -5099,9 +5094,11 @@ pub fn createframe_obj( w_yielding_from: PY_NULL, f_backref: std::ptr::null_mut(), w_builtin, - w_globals, }); - // pyframe.py — final step of __init__. PY_NULL plays the role of + if frame_stores_global { + frame.set_w_globals(w_globals); + } + // pyframe.py `PyFrame.__init__` — final step. PY_NULL plays the role of // Python `None` per the existing `initialize_frame_scopes` convention. // Top-level module / interactive / expression code // arrives here without CO_NEWLOCALS — RustPython codegen emits empty @@ -5380,6 +5377,46 @@ type_id={type_id} frame_forwarded={forwarded} track_young_ptrs={tracks_young}\n" mod tests { use super::load_const_from_code; + #[test] + #[cfg(target_pointer_width = "64")] + fn pyframe_common_layout_has_no_eager_globals_word() { + assert_eq!(std::mem::size_of::(), 112); + } + + #[test] + fn shared_code_stores_only_the_different_globals_on_debugdata() { + // Module code creates debugdata for `w_locals` even in PyPy. Use an + // optimized function code object so this test isolates the uncommon + // globals override that `PyCode.frame_stores_global` controls. + let outer = crate::compile_exec("def f():\n return 1\n").expect("compile"); + let code = super::code_constants(&outer) + .iter() + .find_map(|constant| match constant { + crate::bytecode::ConstantData::Code { code } => Some(code.as_ref()), + _ => None, + }) + .expect("nested function code"); + let w_code = crate::pycode::box_code_constant(code); + let first_globals = pyre_object::w_dict_new(); + let other_globals = pyre_object::w_dict_new(); + + let first = + super::createframe_obj(w_code as *const (), first_globals, std::ptr::null(), None) + .expect("first frame"); + assert!(first.debugdata.is_null()); + assert_eq!(first.get_w_globals(), first_globals); + + let other = + super::createframe_obj(w_code as *const (), other_globals, std::ptr::null(), None) + .expect("shared-code frame"); + assert!(!other.debugdata.is_null()); + assert_eq!(other.get_w_globals(), other_globals); + assert_eq!( + unsafe { crate::w_code_get_w_globals(w_code) }, + first_globals + ); + } + fn nested_code_yields_inside_try(source: &str) -> bool { let outer = crate::compile_exec(source).expect("compile"); let code = super::code_constants(&outer) diff --git a/pyre/pyre-jit-trace/src/descr.rs b/pyre/pyre-jit-trace/src/descr.rs index c693c65abf9..2d73039ec41 100644 --- a/pyre/pyre-jit-trace/src/descr.rs +++ b/pyre/pyre-jit-trace/src/descr.rs @@ -2261,15 +2261,26 @@ static FRAME_DEBUG_DATA_DESCR_GROUP: LazyLock = LazyLock:: pyre_interpreter::pyframe::FRAME_DEBUG_DATA_SIZE, 0, 0, - &[( - "w_locals", - pyre_interpreter::pyframe::FRAME_DEBUG_DATA_W_LOCALS_OFFSET, - std::mem::size_of::(), - Type::Ref, - false, - false, - false, - )], + &[ + ( + "w_globals", + pyre_interpreter::pyframe::FRAME_DEBUG_DATA_W_GLOBALS_OFFSET, + std::mem::size_of::(), + Type::Ref, + false, + false, + false, + ), + ( + "w_locals", + pyre_interpreter::pyframe::FRAME_DEBUG_DATA_W_LOCALS_OFFSET, + std::mem::size_of::(), + Type::Ref, + false, + false, + false, + ), + ], "FrameDebugData", "pyframe::FrameDebugData", ) @@ -2398,17 +2409,6 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { false, false, ), - // `pyframe.py self.w_globals` — the slot the inline - // new-PyFrame helper populates from the function's globals dict. - ( - "PyFrame.w_globals", - crate::frame_layout::PYFRAME_W_GLOBALS_OFFSET, - 8, - Type::Ref, - false, - false, - false, - ), ( "PyFrame.debugdata", crate::frame_layout::PYFRAME_DEBUGDATA_OFFSET, @@ -2427,19 +2427,6 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { false, false, ), - // Inline PyFrame 생성 시 새 frame 의 - // execution_context 슬롯에 caller 의 ec 를 SetfieldGc 로 쓰기 위해 - // 필요. RPython parity 는 interp_jit.py:67 reds=[frame, ec] 의 ec - // 슬롯과 동등 — pyre 는 ec 를 PyFrame 헤더에 inline 저장. - ( - "PyFrame.execution_context", - crate::frame_layout::PYFRAME_EXECUTION_CONTEXT_OFFSET, - 8, - Type::Ref, - false, - false, - false, - ), ( "PyFrame.f_generator_nowref", crate::frame_layout::PYFRAME_F_GENERATOR_NOWREF_OFFSET, @@ -2492,7 +2479,7 @@ static PYFRAME_DESCR_GROUP: LazyLock = LazyLock::new(|| { ( "PyFrame.failed_attr_cleanup", crate::frame_layout::PYFRAME_FAILED_ATTR_CLEANUP_OFFSET, - std::mem::size_of::(), + std::mem::size_of::(), Type::Int, false, false, @@ -3962,6 +3949,12 @@ pub fn rbigint_pair_item1_descr() -> DescrRef { /// `FrameDebugData.w_locals` — the mapping `getorcreatedebug().w_locals` /// reads at the head of `fast2locals` (pyframe.py). pub fn frame_debug_data_w_locals_descr() -> DescrRef { + field_descr_from_group(&FRAME_DEBUG_DATA_DESCR_GROUP, 1) +} + +/// Rare per-frame globals override installed when one PyCode is executed in +/// a namespace other than its first-seen globals. +pub fn frame_debug_data_w_globals_descr() -> DescrRef { field_descr_from_group(&FRAME_DEBUG_DATA_DESCR_GROUP, 0) } @@ -4010,7 +4003,8 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc field_size: usize, field_type: Type, flag: ArrayFlag, - is_immutable: bool| SimpleFieldDescrSpec { + is_immutable: bool, + is_quasi_immutable: bool| SimpleFieldDescrSpec { index: stable_field_index(offset, field_size, field_type, flag == ArrayFlag::Signed), field_key: field_key.to_string(), name: field_key.to_string(), @@ -4018,7 +4012,7 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc field_size, field_type, is_immutable, - is_quasi_immutable: false, + is_quasi_immutable, flag, virtualizable: false, // The group lists only the four read-only payload fields; the @@ -4037,6 +4031,19 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc Type::Int, ArrayFlag::Unsigned, false, + false, + ), + // `pycode.py PyCode._immutable_fields_`: `w_globals?` is filled on + // first execution and then promoted. A shared-code frame using a + // different namespace keeps its override on FrameDebugData instead. + field( + "w_globals", + pyre_interpreter::pycode::CODE_W_GLOBALS_OFFSET, + std::mem::size_of::(), + Type::Ref, + ArrayFlag::Pointer, + false, + true, ), // The slot is written once, by `box_code_object_with_firstlineno`, // onto the object `box_code_object` has just boxed out of a fresh @@ -4049,6 +4056,7 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc Type::Int, ArrayFlag::Signed, true, + false, ), field( "hidden_applevel", @@ -4057,6 +4065,7 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc Type::Int, ArrayFlag::Unsigned, false, + false, ), // `co_name` is absent from `_immutable_fields_`, and this slot is // realized lazily besides: `w_code_name_obj` fills it on first demand, @@ -4068,6 +4077,7 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc Type::Ref, ArrayFlag::Pointer, false, + false, ), ]; specs.sort_by_key(|spec| spec.offset); @@ -4085,28 +4095,41 @@ static PYCODE_DESCR_GROUP: LazyLock = LazyLoc ) }); +fn pycode_field_descr_at(offset: usize) -> DescrRef { + let index = PYCODE_DESCR_GROUP + .field_descrs + .iter() + .position(|d| d.offset() == offset) + .expect("PyCode descr group has no field at requested offset"); + field_descr_from_group(&*PYCODE_DESCR_GROUP, index) +} + pub fn pycode_code_ptr_descr() -> DescrRef { - field_descr_from_group(&*PYCODE_DESCR_GROUP, 0) + pycode_field_descr_at(pyre_interpreter::pycode::CODE_PTR_OFFSET) +} + +pub fn pycode_w_globals_descr() -> DescrRef { + pycode_field_descr_at(pyre_interpreter::pycode::CODE_W_GLOBALS_OFFSET) } /// `PyCode.w_name` — the realized `co_name` string. `w_code_name_obj` builds /// it on first demand and retains it, so the slot IS the getter's value once /// it is non-null; a null slot declines to the residual, which realizes it. pub fn pycode_w_name_descr() -> DescrRef { - field_descr_from_group(&*PYCODE_DESCR_GROUP, 3) + pycode_field_descr_at(pyre_interpreter::pycode::CODE_W_NAME_OFFSET) } /// `PyCode.co_firstlineno_raw` — a signed 32-bit slot, because 3.14's /// `CodeType` constructor accepts zero and negative first lines that /// `CodeObject.first_line_number` cannot hold. pub fn pycode_co_firstlineno_descr() -> DescrRef { - field_descr_from_group(&*PYCODE_DESCR_GROUP, 1) + pycode_field_descr_at(pyre_interpreter::pycode::CODE_CO_FIRSTLINENO_RAW_OFFSET) } /// `PyCode.hidden_applevel` — the frame-hidden flag read by /// `PyFrame.hide()`. pub fn pycode_hidden_applevel_descr() -> DescrRef { - field_descr_from_group(&*PYCODE_DESCR_GROUP, 2) + pycode_field_descr_at(pyre_interpreter::pycode::CODE_HIDDEN_APPLEVEL_OFFSET) } /// Size descriptor for W_IntObject allocation via NewWithVtable. @@ -4981,39 +5004,22 @@ pub fn pyframe_code_descr() -> DescrRef { field_descr_from_group(&PYFRAME_DESCR_GROUP, 3) } -/// R3.3b prep: canonical `PyFrame.w_globals` slot -/// (PYFRAME_W_GLOBALS_OFFSET). Used by -/// `emit_new_pyframe_inline_self_recursive` to populate the -/// W_DictObject sibling so trace-time chases observe a non-null -/// PyObjectRef. `PyFrame.w_globals` is the single globals slot; -/// the raw dict-storage accessor has been retired. -pub fn pyframe_w_globals_obj_descr() -> DescrRef { - field_descr_from_group(&PYFRAME_DESCR_GROUP, 4) -} - -/// rewrite.py handle_call_assembler scalar field read for the +/// rewrite.py `handle_call_assembler` scalar field read for the /// `debugdata` slot of the virtualizable expansion (Phase D-1 prereq). pub fn pyframe_debugdata_descr() -> DescrRef { - field_descr_from_group(&PYFRAME_DESCR_GROUP, 5) -} - -/// PyFrame.execution_context FieldDescr. -/// inline PyFrame 생성 시 caller 의 ec 를 새 frame 으로 SetfieldGc 하기 위해. -/// 호출 사이트는 `helpers.rs::emit_new_pyframe_inline*`. -pub fn pyframe_execution_context_descr() -> DescrRef { - field_descr_from_group(&PYFRAME_DESCR_GROUP, 7) + field_descr_from_group(&PYFRAME_DESCR_GROUP, 4) } pub fn pyframe_f_generator_nowref_descr() -> DescrRef { - field_descr_from_group(&PYFRAME_DESCR_GROUP, 8) + field_descr_from_group(&PYFRAME_DESCR_GROUP, 6) } pub fn pyframe_w_yielding_from_descr() -> DescrRef { - field_descr_from_group(&PYFRAME_DESCR_GROUP, 9) + field_descr_from_group(&PYFRAME_DESCR_GROUP, 7) } pub fn pyframe_f_backref_descr() -> DescrRef { - field_descr_from_group(&PYFRAME_DESCR_GROUP, 10) + field_descr_from_group(&PYFRAME_DESCR_GROUP, 8) } /// `PyFrame.flags` — the byte carrying `FLAG_ESCAPED`. Read-or-written by the @@ -5115,6 +5121,16 @@ mod tests { 1, true, ), + ( + pycode_w_globals_descr(), + "w_globals", + pyre_interpreter::pycode::CODE_W_GLOBALS_OFFSET, + std::mem::size_of::(), + Type::Ref, + false, + 2, + false, + ), ( pycode_hidden_applevel_descr(), "hidden_applevel", @@ -5122,7 +5138,7 @@ mod tests { std::mem::size_of::(), Type::Int, false, - 2, + 3, false, ), ( @@ -5132,7 +5148,7 @@ mod tests { std::mem::size_of::(), Type::Ref, false, - 3, + 4, false, ), ]; @@ -5147,7 +5163,7 @@ mod tests { assert_eq!(field.field_type(), field_type); assert_eq!(field.is_field_signed(), signed); assert_eq!(descr.is_always_pure(), always_pure, "{name}"); - assert!(!descr.is_quasi_immutable()); + assert_eq!(descr.is_quasi_immutable(), name == "w_globals"); assert_eq!(field.index_in_parent(), index_in_parent); let parent = field .get_parent_descr() @@ -5170,7 +5186,7 @@ mod tests { assert_eq!(size.type_id(), pyre_interpreter::pycode::W_CODE_GC_TYPE_ID); assert!(size.is_gc_managed()); assert!(!size.headerless()); - assert_eq!(size.all_fielddescrs().len(), 4); + assert_eq!(size.all_fielddescrs().len(), 5); } #[test] diff --git a/pyre/pyre-jit-trace/src/frame_layout.rs b/pyre/pyre-jit-trace/src/frame_layout.rs index c08dec8246f..aa8e00ed7a8 100644 --- a/pyre/pyre-jit-trace/src/frame_layout.rs +++ b/pyre/pyre-jit-trace/src/frame_layout.rs @@ -32,16 +32,6 @@ pub const PYFRAME_DEBUGDATA_OFFSET: usize = std::mem::offset_of!(PyFrame, debugd /// Byte offset of `lastblock` in `PyFrame`. pub const PYFRAME_LASTBLOCK_OFFSET: usize = std::mem::offset_of!(PyFrame, lastblock); -/// Byte offset of `execution_context` in `PyFrame`. -/// -/// Read/written by `PyreJitState::ec_as_usize` / `set_ec` to mirror the -/// non-vable red inputarg `ec` from `interp_jit.py:67 reds = ['frame', -/// 'ec']`. RPython keeps `ec` outside the vable; pyre's PyFrame happens -/// to carry it inline, so `ec_as_usize` derefs the heap field to read -/// the live ec pointer at JIT entry / extract_live time. -pub const PYFRAME_EXECUTION_CONTEXT_OFFSET: usize = - std::mem::offset_of!(PyFrame, execution_context); - /// Byte offset of `f_generator_nowref` in `PyFrame`. pub const PYFRAME_F_GENERATOR_NOWREF_OFFSET: usize = std::mem::offset_of!(PyFrame, f_generator_nowref); @@ -61,13 +51,6 @@ pub const PYFRAME_F_BACKREF_OFFSET: usize = std::mem::offset_of!(PyFrame, f_back /// pointer. pub const PYFRAME_W_BUILTIN_OFFSET: usize = std::mem::offset_of!(PyFrame, w_builtin); -/// Byte offset of `w_globals` in `PyFrame`. -/// -/// Canonical globals dict object for `frame.w_globals`. The slot is a GCREF -/// and must be visible to the descr / nursery GC walkers so the dict survives -/// across minor collections. -pub const PYFRAME_W_GLOBALS_OFFSET: usize = std::mem::offset_of!(PyFrame, w_globals); - // Backward-compat aliases used by JIT descriptor helpers. pub const PYFRAME_STACK_DEPTH_OFFSET: usize = PYFRAME_VALUESTACKDEPTH_OFFSET; pub const PYFRAME_LOCALS_OFFSET: usize = PYFRAME_LOCALS_CELLS_STACK_OFFSET; @@ -95,7 +78,6 @@ const _: () = { ); assert!(PYFRAME_F_BACKREF_OFFSET == pyre_interpreter::pyframe::PYFRAME_F_BACKREF_OFFSET); assert!(PYFRAME_W_BUILTIN_OFFSET == pyre_interpreter::pyframe::PYFRAME_W_BUILTIN_OFFSET); - assert!(PYFRAME_W_GLOBALS_OFFSET == pyre_interpreter::pyframe::PYFRAME_W_GLOBALS_OFFSET); }; /// Build the virtualizable layout description for `PyFrame`. diff --git a/pyre/pyre-jit-trace/src/helpers.rs b/pyre/pyre-jit-trace/src/helpers.rs index a96402cba4f..204a4d904af 100644 --- a/pyre/pyre-jit-trace/src/helpers.rs +++ b/pyre/pyre-jit-trace/src/helpers.rs @@ -1401,7 +1401,7 @@ pub fn emit_box_float_inline( /// so `handle_new` skips the vtable setfield (rewrite.py:925-933 /// `gen_new_with_vtable` early-out for `vtable == 0`). /// 5. `SetfieldGc` ops for the constructor-visible fields. The non-zero -/// fields (`execution_context`, `pycode`, `w_globals`, +/// fields (`pycode`, /// `locals_cells_stack_w`, `valuestackdepth`, `last_instr=-1`) mirror /// `new_for_call_with_closure`; the nullable GC fields /// (`f_generator_nowref`, `w_yielding_from`, `f_backref`) are written @@ -1435,13 +1435,11 @@ pub fn emit_new_pyframe_inline_with_params( array_size: usize, valuestackdepth: usize, pycode: OpRef, - w_globals: OpRef, - ec: OpRef, + _w_globals: OpRef, ) -> OpRef { use crate::descr::{ - pyframe_code_descr, pyframe_execution_context_descr, pyframe_flags_descr, - pyframe_locals_cells_stack_descr, pyframe_next_instr_descr, pyframe_size_descr, - pyframe_stack_depth_descr, pyframe_w_globals_obj_descr, + pyframe_code_descr, pyframe_flags_descr, pyframe_locals_cells_stack_descr, + pyframe_next_instr_descr, pyframe_size_descr, pyframe_stack_depth_descr, }; use crate::state::pyobject_gcarray_descr; @@ -1500,25 +1498,11 @@ pub fn emit_new_pyframe_inline_with_params( let new_frame = ctx.record_op_with_descr(OpCode::NewWithVtable, &[], pyframe_size_descr()); ctx.heap_cache_mut().new_object(new_frame); - let ec_descr = pyframe_execution_context_descr(); - let ec_idx = ec_descr.index(); - ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_frame, ec], ec_descr); - ctx.heapcache_setfield_cached(new_frame, ec_idx, ec); - let code_descr = pyframe_code_descr(); let code_idx = code_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_frame, pycode], code_descr); ctx.heapcache_setfield_cached(new_frame, code_idx, pycode); - let globals_obj_descr = pyframe_w_globals_obj_descr(); - let globals_obj_idx = globals_obj_descr.index(); - ctx.record_op_with_descr( - OpCode::SetfieldGc, - &[new_frame, w_globals], - globals_obj_descr, - ); - ctx.heapcache_setfield_cached(new_frame, globals_obj_idx, w_globals); - let locals_descr = pyframe_locals_cells_stack_descr(); let locals_idx = locals_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_frame, locals_array], locals_descr); @@ -1565,13 +1549,11 @@ pub fn emit_new_pyframe_inline_self_recursive( array_size: usize, valuestackdepth: usize, pycode: OpRef, - w_globals: OpRef, - ec: OpRef, + _w_globals: OpRef, ) -> OpRef { use crate::descr::{ - pyframe_code_descr, pyframe_execution_context_descr, pyframe_flags_descr, - pyframe_locals_cells_stack_descr, pyframe_next_instr_descr, pyframe_size_descr, - pyframe_stack_depth_descr, pyframe_w_globals_obj_descr, + pyframe_code_descr, pyframe_flags_descr, pyframe_locals_cells_stack_descr, + pyframe_next_instr_descr, pyframe_size_descr, pyframe_stack_depth_descr, }; use crate::state::pyobject_gcarray_descr; @@ -1613,27 +1595,12 @@ pub fn emit_new_pyframe_inline_self_recursive( // the explicit assignments inside `new_for_call_with_closure`. // Order matches the field declaration so the optimizer's lazy-set // replace logic groups them together. - let ec_descr = pyframe_execution_context_descr(); - let ec_idx = ec_descr.index(); - ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_frame, ec], ec_descr); - ctx.heapcache_setfield_cached(new_frame, ec_idx, ec); - // `pycode` arrives as a trace-time Ref Const (the bound `PyCode`). let code_descr = pyframe_code_descr(); let code_idx = code_descr.index(); ctx.record_op_with_descr(OpCode::SetfieldGc, &[new_frame, pycode], code_descr); ctx.heapcache_setfield_cached(new_frame, code_idx, pycode); - // pyframe.py:49 `self.w_globals = w_globals` — store the canonical dict. - let globals_obj_descr = pyframe_w_globals_obj_descr(); - let globals_obj_idx = globals_obj_descr.index(); - ctx.record_op_with_descr( - OpCode::SetfieldGc, - &[new_frame, w_globals], - globals_obj_descr, - ); - ctx.heapcache_setfield_cached(new_frame, globals_obj_idx, w_globals); - // `locals_array` is a fresh `NewArrayClear` op result. PyPy's // executor-while-trace model would have `Box.value` carry the // actual allocated array ref; pyre's `record_op` does not execute diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index 1c2af36fac0..8dcd9416918 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -83,21 +83,11 @@ pub(crate) fn carrier_ec_leave( if concrete_frame == 0 || concrete_ec.is_null() { return; } - // `frame.execution_context`, the same read `walker_ec_enter`'s counterpart - // performs at the inlined-call push. - let callee_ec = ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[root_sym.frame()], - crate::descr::pyframe_execution_context_descr(), - ); - // Every `GetfieldGcR` the enter/leave pair records carries its concrete - // value (`history.py *FrontendOp(pos, value)`); without it - // `concrete_of_opref` reports this result symbolic on the residual-call - // and snapshot paths. The value is the same EC the leave below acts on. - ctx.set_opref_concrete( - callee_ec, - majit_ir::Value::Ref(majit_ir::GcRef(concrete_ec as usize)), - ); + // All inlined frames share the portal's separately carried EC red. + let callee_ec = root_sym.execution_context(); + if callee_ec.is_none() { + return; + } super::inline_call::walker_ec_leave( ctx, callee_frame, @@ -172,9 +162,8 @@ pub fn dispatch_via_miframe( // `fbw_mode.snapshot_sym` is non-null on every default-JIT run. // Recover the portal EC red off `sym.frame` before the first opcode is // dispatched (thus before any guard is recorded), caching it into - // `sym.execution_context`. A bridge-from-guard sym whose ec color collides - // with a real frame slot is left `OpRef::NONE` by `setup_bridge_sym`, which - // defers recovery to `ensure_execution_context`. The walker's + // `sym.execution_context`. Bridge setup decodes the dedicated portal EC + // color into this field before the walk begins. The walker's // snapshot-capture path runs `collect_outer_active_boxes` AFTER the guard, // so recovering there would record the getfield after the guard that // references it (use-before-def). Seed here — the trait's pre-guard @@ -943,7 +932,6 @@ pub(crate) fn recipe_parent_frame_from_recipe( recipe, root_ec, root_ec_box, - root_frame_box, Vec::new(), )?; let frame_box = pending.sym.frame(); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs index f65cc1ed1b4..319da2787e7 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/fbw_state.rs @@ -86,6 +86,45 @@ pub(crate) fn fbw_inline_recursion_count( .count() } +/// Total inline-stack bound for a callee after its same-greenkey recursion +/// count has already been checked against the live `max_unroll_recursion`. +/// +/// PyPy `_opimpl_recursive_call` has no second, total-framestack depth gate: +/// once a recursive portal is admitted, `memory_manager.max_unroll_recursion` +/// is the sole value-returning recursion bound. FBW's generic multiframe cap +/// is a local cost valve for non-recursive call chains and must not silently +/// turn an upstream value of 7 into an effective 6 when an ambient inline +/// frame is present. Raising chains retain their separate depth-two safety +/// bound because their carrier unwind crosses suspended frames. +pub(crate) fn fbw_effective_multiframe_depth( + contains_raise: bool, + inline_recursion_count: usize, +) -> usize { + if contains_raise { + 2 + } else if inline_recursion_count != 0 { + usize::MAX + } else { + fbw_max_multiframe_depth() + } +} + +#[cfg(test)] +mod recursion_depth_policy_tests { + use super::*; + + #[test] + fn value_returning_recursion_is_bounded_only_by_max_unroll_recursion() { + assert_eq!(fbw_effective_multiframe_depth(false, 1), usize::MAX); + assert_eq!(fbw_effective_multiframe_depth(false, 7), usize::MAX); + } + + #[test] + fn raising_recursion_keeps_the_carrier_unwind_safety_bound() { + assert_eq!(fbw_effective_multiframe_depth(true, 1), 2); + } +} + /// The innermost inline level's strict-fold frame register (`u16::MAX` when /// inactive / no inline level). pub(crate) fn fbw_strict_fold_frame_reg(ctx: &WalkContext<'_, '_, Sym>) -> u16 { @@ -1808,23 +1847,16 @@ pub(crate) fn fbw_abort_nested_unjournaled_residual( // letting the enclosing loop retrace (`pyjitpl.py:2818-2828`). if let Some((callee_code_key, _)) = hazardous_callee { fbw_deny_hazardous_inline(callee_code_key); - // `fbw_deny_hazardous_inline` writes a thread-local set that only - // this walker reads, so the deny stayed invisible to the warm - // state: `dont_trace_here` counted zero on every fixture that - // reaches here. The upstream answer cited above is - // `disable_noninlinable_function(greenkey_of_huge_function)` - // (`pyjitpl.py:2821`), which sets `JC_DONT_TRACE_HERE` on the - // callee's cell (`warmstate.py:331-337`). The recursion-bound deny - // in `inline_call.rs` already calls it for the callee it names; - // this one names a callee the same way and had no reason not to. + // `fbw_deny_hazardous_inline` writes the walker-local policy set; + // upstream also applies `disable_noninlinable_function` to the + // callee's own greenkey. Use the typed key so the verdict lands + // on the same comparekey-bearing cell consulted at the next call. if let Some((driver, _)) = crate::driver::try_driver_pair() { + let key = crate::driver::make_green_key_typed(callee_code_key as *const (), 0); driver .meta_interp_mut() .warm_state_mut() - .disable_noninlinable_function(crate::driver::make_green_key( - callee_code_key as *const (), - 0, - )); + .disable_noninlinable_function_for_key(&key); } } // The flush this latch feeds resumes the OUTERMOST caller at the CALL diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs index 8d2bbd88ea1..e35b7b066be 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/heapcache_ops.rs @@ -687,6 +687,5 @@ pub(crate) fn getfield_gc_via_heapcache( } /// `virtualizable_gen.rs` pyre PyFrame static-field order -/// `[last_instr, pycode, valuestackdepth, debugdata, w_globals]`. +/// `[last_instr, pycode, valuestackdepth, debugdata]`. pub(crate) const VABLE_CODE_FIELD_IDX: usize = 1; -pub(crate) const VABLE_NAMESPACE_FIELD_IDX: usize = 4; diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 089cfdc0c7d..48ae4e4b45d 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -108,13 +108,9 @@ unsafe fn positional_defaults_for_inline( /// Path-1 (#68): resolve a scalar `getfield_vable_r` read off an inlined /// callee's OWN (unseeded) portal frame to the callee's compile-time -/// constant. This is the walk-time mirror of the codewriter's non-portal -/// branch (`codewriter.rs` LOAD_CONST/LOAD_GLOBAL): -/// a non-portal callee's `pycode`/`w_globals` are constants fed as -/// `ConstRef`, never read off the portal frame reg (which, when inlined, -/// aliases the caller's frame and would read the wrong field). Only the -/// Ref-typed `pycode` (field 1) and `w_globals` (field 5) carry a -/// compile-time constant; Int frame state (`last_instr`, `valuestackdepth`) +/// constant. Only the Ref-typed `pycode` (field 1) remains a concrete +/// virtualizable field; globals is derived through PyFrame.get_w_globals. +/// Int frame state (`last_instr`, `valuestackdepth`) /// does not. Returns `None` when not an inline sub-walk, the field is not /// resolvable, or the layout is absent — callers fall through to the /// `VableBoxNotSeeded` error (such callees are declined up-front by @@ -142,7 +138,6 @@ pub(crate) fn try_resolve_inline_callee_static_field( } }; let const_ptr = match field_idx { - VABLE_NAMESPACE_FIELD_IDX => consts.w_globals, VABLE_CODE_FIELD_IDX => consts.w_code, _ => return Ok(None), }; @@ -323,7 +318,7 @@ pub(crate) fn inline_resolvable_static_vable_read( }; matches!( info.static_field_by_descr(descr), - Some(VABLE_CODE_FIELD_IDX) | Some(VABLE_NAMESPACE_FIELD_IDX) + Some(VABLE_CODE_FIELD_IDX) ) } @@ -887,7 +882,8 @@ fn foreign_callee_admits_call_assembler(w_code: *const ()) -> bool { /// #62: full-body-walk direct `CALL_ASSEMBLER` for a self-recursive call /// at the inline recursion-bound boundary. /// -/// When the FBW inline depth for a callee reaches `FBW_MAX_INLINE_RECURSION` +/// When the FBW inline depth for a callee reaches the warmstate's live +/// `max_unroll_recursion` /// the call would otherwise degrade to a generic may-force residual, which /// re-enters the callee through the func-entry residency door — one /// heavyweight frame build + entry-bridge per recursive call (the @@ -1122,9 +1118,9 @@ pub(crate) fn try_walker_call_assembler_self_recursive( } { return Ok(None); } - // Resolve the callee's own loop or trace-in-progress marker with - // `make_green_key(w_callee_code, 0)` (`pc = 0` = function entry). A - // pending token only proves the callee is + // Resolve the callee's own loop or trace-in-progress marker with the live + // W_Code object used by `PyFrame.pycode`, the portal greens, and + // `JitCell.comparekey`. A pending token only proves the callee is // being traced; emission below resolves compiled-or-tmp so the descr never // carries a bodyless token. let (driver, _) = crate::driver::driver_pair(); @@ -1173,22 +1169,17 @@ pub(crate) fn try_walker_call_assembler_self_recursive( param_boxes.push(crate::state::wrapint(ctx.trace_ctx, raw_arg)); } - // Execution-context red: recover it fresh off the materialized caller - // portal frame via `GETFIELD_GC_R(frame, execution_context_descr)` rather - // than trusting the seeded `sym.execution_context` OpRef. The seeded OpRef - // is a bridge-decode color-bank value (`setup_bridge_sym`) that is - // concrete-correct at forward-compile but rebinds to the callee's own - // `pycode` when this compiled self-recursive trace re-enters as a NESTED - // bridge, building the callee frame with `ec == pycode` and faulting later - // in `frame_builtin`. The outer portal frame's `execution_context` field - // is always the true ec (single ExecutionContext, boot-pinned), so reading - // it off `caller_frame` is the nested-resume-safe source — the same - // recovery `ensure_execution_context` (`trace_opcode.rs`) performs. - let ec = ctx.trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[caller_frame], - crate::descr::pyframe_execution_context_descr(), - ); + // `ec` is the portal's second red (`interp_jit.py reds=['frame', 'ec']`). + // It is shared by every recursive MIFrame and never recovered from the + // app-level frame object. + let sym_ptr = ctx.fbw_mode.snapshot_sym; + if sym_ptr.is_null() { + return Ok(None); + } + let ec = unsafe { &*sym_ptr }.execution_context(); + if ec.is_none() { + return Ok(None); + } // Build the callee PyFrame inline (Branch A): a single positional // local, no cells, constant code / globals. @@ -1203,7 +1194,6 @@ pub(crate) fn try_walker_call_assembler_self_recursive( nlocals, pycode_const, w_globals_obj_const, - ec, ); // do_residual_call step 1 (`pyjitpl.py`): FORCE_TOKEN + @@ -3476,21 +3466,36 @@ fn try_walker_inline_resolved_user_call_inner( // later sibling calls in the same trace go straight to CALL_ASSEMBLER // instead of starting a fresh unroll from their now-shallower framestack. let callee_code_key = w_code as pyre_object::PyObjectRef as usize; - let callee_green_key = crate::driver::make_green_key(w_code, 0); + let callee_green_key = crate::driver::make_green_key_typed(w_code, 0); if let Some((driver, _)) = crate::driver::try_driver_pair() && !driver .meta_interp_mut() .warm_state_mut() - .can_inline_callable(callee_green_key) + .can_inline_callable_for_key(&callee_green_key) { return resolved_inline_decline(op.pc, line!()); } - if fbw_inline_recursion_count(ctx, callee_code_key) >= FBW_MAX_INLINE_RECURSION { + // `_opimpl_recursive_call` (`pyjitpl.py`) reads + // `memory_manager.max_unroll_recursion` at this decision. Do not bake the + // RPython default here: `pypyjit.set_param(max_unroll_recursion=...)` + // mutates the live warmstate and must steer the production FBW path too. + // A skeleton walk has no installed driver, so only that diagnostic path + // falls back to the upstream default. + let max_unroll_recursion = crate::driver::try_driver_pair() + .map(|(driver, _)| { + driver + .meta_interp_mut() + .warm_state_mut() + .max_unroll_recursion() as usize + }) + .unwrap_or(FBW_DEFAULT_MAX_INLINE_RECURSION); + let inline_recursion_count = fbw_inline_recursion_count(ctx, callee_code_key); + if inline_recursion_count >= max_unroll_recursion { if let Some((driver, _)) = crate::driver::try_driver_pair() { driver .meta_interp_mut() .warm_state_mut() - .disable_noninlinable_function(callee_green_key); + .disable_noninlinable_function_for_key(&callee_green_key); } return resolved_inline_decline(op.pc, line!()); } @@ -3625,40 +3630,14 @@ fn try_walker_inline_resolved_user_call_inner( && args_all_builtin_integer && fbw_callee_body_has_binary_op_residual(body.code, callee_descr_refs) { - // A depth-1 carrier-resume sub-walk (`drive_bridge_frame_subwalk`) drives - // its reconstructed frame as the sub-walk root with an empty framestack, - // so it is uniform with a plain root bridge for the multiframe seed — - // admit it too so a rare guard-bridge continuation inlines its nested - // int-arith calls instead of residualizing them (the gh#343 branchy-callee - // cost). - // The nested levels are admitted on the same terms. - // `opimpl_recursive_call` (`pyjitpl.py:1390-1416`) makes the inline - // decision from the portal-frame count alone — already applied above via - // `fbw_inline_recursion_count` — and asks nothing about how deep the - // framestack is or whether a bridge or a primary trace is walking. - // - // An inline sub-walk is the exception, and it is one with no upstream - // counterpart: `perform_call` pushes onto `MetaInterp.framestack` and - // returns to the single `interpret()` loop, so upstream is never inside - // one walk while starting another and can never seed a multiframe - // snapshot from a sub-walk. Seeding one here makes the wasm bridge - // replay re-execute the sub-walk's body — wrong output from - // `synth/ca_bridge_multiframe_resume_double_call` and - // `synth/recursion_memo_branch`, and a `fib_recursive` timeout, all - // while dynasm and cranelift stay green. - let root_bridge = !ctx.fbw_mode.carrier_resume - && !ctx.fbw_mode.inline_subwalk - && !ctx.fbw_mode.snapshot_sym.is_null(); - // A carrier-resume sub-walk (`drive_bridge_frame_subwalk`) drives its - // reconstructed frame(s) forward through the same metainterp the initial - // trace uses, so its inline of a nested int-arith call is the SAME as a - // primary trace's — admit it (any inline depth) so the rare guard-bridge - // continuation inlines instead of residualizing. - let subwalk_admit = ctx.fbw_mode.carrier_resume && !ctx.fbw_mode.snapshot_sym.is_null(); - let safe_root_bridge = root_bridge || subwalk_admit; - if !safe_root_bridge { - return resolved_inline_decline(op.pc, line!()); - } + // `_opimpl_recursive_call` runs in the one `MetaInterp.interpret` + // loop after `perform_call` pushes each MIFrame; it has no separate + // "root bridge may inline, nested bridge sub-walk may not" gate. FBW + // now carries the same frame chain explicitly: `InlineFrameGuard` + // keeps this callee and its `parent_frame` live while the nested walk + // runs, and multiframe guard snapshots encode that chain. Refusing a + // nested root-bridge call here dropped the parent's continuation from + // the compiled shape and capped fib one unroll short of PyPy. bridge_rec_root_selfrec = unsafe { let raw = pyre_interpreter::w_code_get_ptr(w_code as pyre_object::PyObjectRef) as *const pyre_interpreter::CodeObject; @@ -3670,13 +3649,15 @@ fn try_walker_inline_resolved_user_call_inner( // the callee's `CodeObject` (self-recursive), so re-inlining // it rebuilds the identical framestack and reaches the identical abort — // the enclosing loop is retired for a decline that belongs to the callee. - // `warmstate.py` `disable_noninlinable_function` is the same answer: - // the flag it sets means "do not inline calls to this function", and the - // enclosing loop is left free to retrace. + // `WarmEnterState::disable_noninlinable_function_for_key` carries the + // general answer: its flag means "do not inline calls to this function", + // and the enclosing loop is left free to retrace. // // `bridge_rec_root_selfrec` is exempt: that admission carries its own - // `SELFREC_CA_FOLD_ACTIVE` exemption from the hazard arm (:2696), so its - // recursive residual is not what named the callee here. + // `SELFREC_CA_FOLD_ACTIVE` exemption from + // `fbw_abort_nested_unjournaled_residual`, so its recursive residual is + // not what named the callee here. The warmstate verdict is deferred to + // this same point so its typed lookup cannot hide the exemption. if !bridge_rec_root_selfrec && fbw_hazardous_inline_denied(callee_code_key) { return resolved_inline_decline(op.pc, line!()); } @@ -3991,11 +3972,8 @@ fn try_walker_inline_resolved_user_call_inner( // `guard_failures` from 937 to 7408 — because the unwind then crosses two // suspended copies of the same frame, the shape `fbw_max_rec_unroll_depth` // bounds above. - let effective_multiframe_depth = if contains_raise { - 2 - } else { - fbw_max_multiframe_depth() - }; + let effective_multiframe_depth = + fbw_effective_multiframe_depth(contains_raise, inline_recursion_count); // The instance-`__next__` FOR_ITER route uses the same seeded-frame shape // as other CALL-entered inlines. Its catch arm owns exception-to-exhaustion // conversion, so neither replay safety nor an unseeded caller-boundary @@ -4088,14 +4066,28 @@ fn try_walker_inline_resolved_user_call_inner( callable_guard_value = bound.function; } - // Path-1 (#68): the inlined callee's compile-time-constant frame fields, - // so a scalar `getfield_vable_r` off its own (unseeded) portal frame — - // the `w_globals` namespace for a LOAD_GLOBAL, the promote-to-const - // `pycode` — resolves to the constant via - // `try_resolve_inline_callee_static_field` instead of aborting - // `VableBoxNotSeeded`. Mirror of the codewriter non-portal branch. + // `PyCode.frame_stores_global`: the common globals identity lives on the + // code object and needs no frame field. A shared code object rebound to a + // different namespace creates `FrameDebugData.w_globals` in PyPy. This + // custom inline-frame encoder does not yet put that rare debugdata object + // into resume data, so keep the call residual instead of compiling a + // frame whose `get_w_globals()` would silently read the first namespace. + let callee_globals_obj = unsafe { pyre_interpreter::function_get_globals_obj(callable) }; + if unsafe { + pyre_interpreter::w_code_frame_stores_global( + w_code as pyre_object::PyObjectRef, + callee_globals_obj, + ) + } { + return resolved_inline_decline(op.pc, line!()); + } + + // Path-1 (#68): the inlined callee's promoted `pycode` static field and + // its code-derived globals semantic constant. The latter is no longer a + // physical virtualizable scalar; LOAD_GLOBAL derives it through the + // callee frame's pycode. let inline_consts = InlineCalleeConsts { - w_globals: unsafe { pyre_interpreter::function_get_globals_obj(callable) } as usize, + w_globals: callee_globals_obj as usize, w_code: callee_code_key, jitcode_index: crate::state::ensure_jitcode_index(callee_code_key as *const ()) .map_or(-1, |index| index as i32), @@ -4642,13 +4634,8 @@ fn try_walker_inline_resolved_user_call_inner( break 'seed; } - // ec red: the shared ExecutionContext (perform_call threads the - // caller's ec down). Recover it off the materialized caller portal - // frame via `GETFIELD_GC_R` rather than the seeded - // `sym.execution_context` OpRef — the seeded OpRef rebinds to the - // callee's own `pycode` when this compiled trace re-enters as a nested - // bridge (see `try_walker_call_assembler_self_recursive`). The outer - // portal frame's `execution_context` field is the single true ec. + // ec red: `perform_call` threads the caller's second portal red + // down unchanged, just as PyPy shares `ec` between MIFrames. let sym_ptr = ctx.fbw_mode.snapshot_sym; if sym_ptr.is_null() { if try_multiframe { @@ -4658,11 +4645,14 @@ fn try_walker_inline_resolved_user_call_inner( break 'seed; } let sym = unsafe { &*sym_ptr }; - let callee_ec = ctx.trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[sym.frame()], - crate::descr::pyframe_execution_context_descr(), - ); + let callee_ec = sym.execution_context(); + if callee_ec.is_none() { + if try_multiframe { + return resolved_inline_decline(op.pc, line!()); + } + seed_break_reason = "Collapse::NoExecutionContextRed"; + break 'seed; + } let pycode_const = ctx.trace_ctx.const_ref(w_code as i64); let w_globals_obj_const = ctx.trace_ctx.const_ref(inline_consts.w_globals as i64); @@ -4676,7 +4666,6 @@ fn try_walker_inline_resolved_user_call_inner( nlocals + ncells, pycode_const, w_globals_obj_const, - callee_ec, ); callee_regs_r[frame_reg as usize] = callee_frame; @@ -4728,7 +4717,14 @@ fn try_walker_inline_resolved_user_call_inner( // host handle; the frontend op above keeps the frame reachable. drop(frame); callee_regs_r[ec_reg as usize] = callee_ec; - callee_concrete_r[ec_reg as usize] = ConcreteValue::Null; + // `perform_call` threads the same concrete ExecutionContext into + // every MIFrame. The symbolic second red above and its concrete + // shadow are one value; leaving only the shadow unknown makes + // `build_single_frame_miframe` reject an otherwise complete + // callee image during an escape, after which the legacy caller + // replay resumes past CALL without its result. + callee_concrete_r[ec_reg as usize] = + ConcreteValue::Ref(concrete_ec as pyre_object::PyObjectRef); // Retain for a possible `SubLoopCalleeCallAssembler` emit. ca_callee_frame = callee_frame; @@ -4992,7 +4988,7 @@ fn try_walker_inline_resolved_user_call_inner( // `newframe` pushes (`pyjitpl.py, 1862-1874`) is the tracer's // `MIFrame` instead, and is not what `enter` takes its vref of. let entered_ec = callee_frame_seeded && !ca_concrete_frame.is_null() && { - let concrete_ec = unsafe { (*ca_concrete_frame).execution_context } + let concrete_ec = pyre_interpreter::call::getexecutioncontext() as *mut pyre_interpreter::PyExecutionContext; if concrete_ec.is_null() { false @@ -5200,14 +5196,11 @@ fn try_walker_inline_resolved_user_call_inner( // `disable_noninlinable_function` is applied to. let subwalk_jd_no = crate::state::note_inline_subwalk_start( ( - crate::driver::make_green_key(raw_callee_code as *const (), 0), + crate::driver::make_green_key(w_code, 0), // The callee's greens are in scope here, so the log carries // them: `disable_noninlinable_function` applies to this key, // and it reaches a cell. - Some(crate::driver::make_green_key_typed( - raw_callee_code as *const (), - 0, - )), + Some(crate::driver::make_green_key_typed(w_code, 0)), ), sub_wc.trace_ctx.get_trace_position(), ); @@ -5449,7 +5442,7 @@ fn try_walker_inline_resolved_user_call_inner( // completes, so every callee exit — return, exception, or decline — // arrives here before any of the early returns below. if entered_ec { - let concrete_ec = unsafe { (*ca_concrete_frame).execution_context } + let concrete_ec = pyre_interpreter::call::getexecutioncontext() as *mut pyre_interpreter::PyExecutionContext; // `leave(frame, w_exitvalue, got_exception)` — the caller passes true // only when the frame is unwinding an exception, which for an inlined diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index f71c6b953aa..30f57b23e8c 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -492,7 +492,7 @@ impl CalleeLocalsShadow { /// One inlined-callee level of the walk's framestack. pub struct InlineFrame { /// Callee `w_code`, used by the recursion-depth scan. Once the same code - /// reaches [`FBW_MAX_INLINE_RECURSION`], the call folds to a residual + /// reaches the live `max_unroll_recursion`, the call folds to a residual /// instead of unrolling its call tree (`pyjitpl.py`). pub w_code: usize, /// Paused levels sitting between this callee and the next one out, @@ -1404,8 +1404,7 @@ fn record_fresh_application_traceback( /// Compile-time-constant frame fields of an inlined callee. #[derive(Clone, Copy)] pub struct InlineCalleeConsts { - /// `frame.w_globals` object (`VABLE_NAMESPACE_FIELD_IDX` = 4): the - /// callee function's `__globals__` as a `PyObjectRef`. + /// The callee function's `__globals__` as a `PyObjectRef`. w_globals: usize, /// `frame.pycode` (`VABLE_CODE_FIELD_IDX` = 1): the callee's `W_Code` /// pointer. @@ -4839,12 +4838,14 @@ fn guard_current_frame_globals_identity( if let Some(consts) = ctx.inline_callee_consts { return Ok(consts.w_globals == expected_globals as usize); } - let Some(w_globals_op) = ctx - .trace_ctx - .virtualizable_box_at(VABLE_NAMESPACE_FIELD_IDX) - else { + let sym_ptr = ctx.fbw_mode.snapshot_sym; + if sym_ptr.is_null() { return Ok(false); - }; + } + let w_globals_op = unsafe { (&*sym_ptr).frame_w_globals() }; + if w_globals_op.is_none() { + return Ok(false); + } let expected = ctx.trace_ctx.const_ref(expected_globals as i64); if w_globals_op.is_constant() { return Ok(ctx.trace_ctx.const_value(w_globals_op) == Some(expected_globals as i64)); @@ -4865,21 +4866,15 @@ fn guard_current_frame_globals_identity( fn replace_movable_load_global_namespace_with_frame_globals( ctx: &mut WalkContext<'_, '_, Sym>, - ei: &majit_ir::EffectInfo, - allboxes: &mut [OpRef], + pyre_helper: majit_ir::PyreHelperKind, + r_args: &mut [OpRef], ) { - if ei.pyre_helper != majit_ir::PyreHelperKind::LoadGlobal { + if pyre_helper != majit_ir::PyreHelperKind::LoadGlobal { return; } - let Some(ns_box) = allboxes.get_mut(1) else { + let Some(ns_box) = r_args.first_mut() else { return; }; - let Some(Value::Ref(majit_ir::GcRef(ns_ptr))) = ctx.trace_ctx.box_value(*ns_box) else { - return; - }; - if !majit_gc::can_move(majit_ir::GcRef(ns_ptr)) { - return; - } // `pyframe.py:128-132 LOAD_GLOBAL` asks the current MIFrame for // `self.get_w_globals()`. An inline sub-walk therefore has to source the @@ -4900,9 +4895,22 @@ fn replace_movable_load_global_namespace_with_frame_globals( .filter(|frame| !frame.is_none()) else { // The old single-frame inline path has no callee red to read. Do - // not borrow the root frame: the residual's unbound callee-frame - // argument will decline this trace instead of compiling the wrong - // namespace. + // not borrow the root frame — that collapses caller and callee + // identity — but the callee's OWN `__globals__` is recorded right + // here, and `try_walker_load_global_cell_fold`'s builtins leg is + // written for exactly this case (`frame_ptr == 0`, deriving the + // builtin module from the namespace's `__builtins__` cell). Naming + // it directly is what lets that leg run; leaving the codewriter's + // null placeholder standing declines the fold and then aborts the + // trace on the residual's unbound callee-frame argument. + // + // Immovable only, matching `guard_current_frame_globals_identity`, + // where both fold legs end: a movable namespace declines there + // anyway, so substituting one would buy nothing and bake a pointer + // the GC may forward into the surviving residual. + if consts.w_globals != 0 && !majit_gc::can_move(majit_ir::GcRef(consts.w_globals)) { + *ns_box = ctx.trace_ctx.const_ref(consts.w_globals as i64); + } return; }; let w_globals = crate::state::frame_get_globals_obj(ctx.trace_ctx, frame); @@ -4915,10 +4923,12 @@ fn replace_movable_load_global_namespace_with_frame_globals( return; } - if let Some(w_globals_op) = ctx - .trace_ctx - .virtualizable_box_at(VABLE_NAMESPACE_FIELD_IDX) - { + let sym_ptr = ctx.fbw_mode.snapshot_sym; + if sym_ptr.is_null() { + return; + } + let w_globals_op = unsafe { (&*sym_ptr).frame_w_globals() }; + if !w_globals_op.is_none() { *ns_box = w_globals_op; } } @@ -5701,17 +5711,23 @@ fn collect_outer_active_boxes( active.push(m); continue; } - // The mirror did not supply this operand-stack slot, so the - // decoded edge-move recovery below is the only other - // per-slot source. Without an entry for this color the - // fallback reads the resume merge color out of the guard-pc - // register file, where it is unwritten at the guard point or - // reused by the regalloc — the #424 staleness the per-slot - // mirror was introduced to replace. The kept-stack gate - // admits a valid mirror on the stated grounds that this - // recovery covers such a slot; report the case where that is - // false so the caller declines rather than encode the stale - // read. + // The mirror did not supply this operand-stack slot. Two + // other per-slot sources remain, and only the absence of + // BOTH is a hole: the decoded edge-move recovery + // (`kept_recovered`, tested below) and the virtualizable + // shadow, which the resolution further down already treats + // as authoritative for a slot the guard pc's color map does + // not claim (`shadow_is_real` → `vbox`). Where neither + // answers, the fallback reads the resume merge color out of + // the guard-pc register file, unwritten at the guard point + // or reused by the regalloc — the #424 staleness the + // per-slot mirror replaced; report that case so the caller + // declines rather than encode the stale read. Declining + // where the shadow does answer is not free: the capture + // point sits after a residual has run, so it lands as a + // `fbw_rolled_back_with_effects` double-apply + // (`raise_reg_unbound_jitstress`, mirror at depth 1 against + // a guard resuming at depth 3). // // Restricted to a NONE hole, where there is demonstrably no // source at all. A NULL const-ptr is NOT declined, and that @@ -5730,8 +5746,17 @@ fn collect_outer_active_boxes( // and the vable write-through store one directly. Settling // that is what closes the remaining case. if m == OpRef::NONE && !kept_recovered.contains_key(&idx) { - if let Some(first) = unrecovered_kept.as_deref_mut() { - first.get_or_insert(idx); + let shadow_sources_slot = trace_ctx + .virtualizable_box_at(crate::virtualizable_gen::NUM_VABLE_SCALARS + sem) + .is_some_and(|b| { + b != OpRef::NONE + && !opref_is_null_const_ptr(b) + && b.ty() == Some(majit_ir::Type::Ref) + }); + if !shadow_sources_slot { + if let Some(first) = unrecovered_kept.as_deref_mut() { + first.get_or_insert(idx); + } } } } @@ -5782,40 +5807,7 @@ fn collect_outer_active_boxes( .filter(|&v| v != OpRef::NONE && !opref_is_null_const_ptr(v)); let red_field = if color as u16 == portal_frame_reg { sym.frame() - } else if !sym.execution_context().is_none() { - // EC red already seeded on this snapshot path. - sym.execution_context() - } else if !sym.frame().is_none() { - // Adapter / inline-caller snapshot path leaves - // `sym.execution_context` unseeded (`OpRef::NONE`). This is the - // pre-guard inline-parent-frame collection (the paused caller's - // active boxes are built BEFORE the callee sub-walk records its - // guards), so recording the recovery getfield here is - // well-ordered. Recover the EC from the frame the same way the - // `MIFrame::ensure_execution_context` does - // (trace_opcode.rs): record `getfield - // frame.execution_context` and route that OpRef through as the - // portal EC red, so the resume snapshot never pushes NONE for - // `interp_jit.py reds = ['frame', 'ec']`. A NONE EC escapes - // as a null execution-context pointer and SIGSEGVs (rc=139) or - // trips the Ref-bank NONE guard (rc=101). - // - // The post-guard snapshot-capture path - // (`walker_capture_snapshot_for_last_guard`) reaches this fn with - // the outer full-body `sym`, whose EC is eagerly recovered at - // walk entry (`seed_execution_context_for_walk`) — so on that - // path `sym.execution_context` is already real above and this - // branch (which would record AFTER the guard, a use-before-def) - // is not taken. - trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[sym.frame()], - crate::descr::pyframe_execution_context_descr(), - ) } else { - // Neither EC nor frame is recoverable: keep the raw NONE so the - // downstream Ref-bank NONE guard surfaces the unrecoverable case - // instead of silently masking it. sym.execution_context() }; live_reg @@ -6080,9 +6072,12 @@ pub(crate) enum GuardStampTarget { GuardFromEnd(usize), } -/// `rlib/jit.py` `max_unroll_recursion` default (= warmstate -/// `DEFAULT_MAX_UNROLL_RECURSION`). -const FBW_MAX_INLINE_RECURSION: usize = 7; +/// Fallback for `rlib/jit.py` `max_unroll_recursion` when a skeleton walk has +/// no live driver/warmstate to consult. Production must read the parameter +/// from warmstate: `pypyjit.set_param(max_unroll_recursion=...)` updates it at +/// runtime, and `_opimpl_recursive_call` (`pyjitpl.py`) compares against that +/// live memory-manager value rather than the default. +const FBW_DEFAULT_MAX_INLINE_RECURSION: usize = 7; /// Upper bound on the parameter count the self-recursive `CALL_ASSEMBLER` fold /// accepts. Accumulator/linear recursion is low-arity; the cap keeps a @@ -7828,12 +7823,8 @@ fn walker_concrete_ref_object( } } -/// Resolve the walker's execution-context OpRef from the outer -/// portal `sym.frame` (via [`fbw_mode.snapshot_sym`]), recovering it off -/// the frame with `GetfieldGcR(frame, execution_context)` when -/// `sym.execution_context` is unseeded. Mirrors the inline-frame EC -/// recovery (jitcode_dispatch.rs `try_walker_inline_self_recursive`) and -/// `MIFrame::ensure_execution_context` (`trace_opcode.rs`). +/// Resolve the walker's execution-context OpRef from the outer portal's +/// separately carried second red (`interp_jit.py reds = ['frame', 'ec']`). /// /// `None` outside a production full-body walk (no materialized portal sym) /// or when the portal frame OpRef is unset — the PUSH_EXC_INFO / POP_EXCEPT @@ -7846,31 +7837,16 @@ fn walker_ensure_execution_context( return None; } let sym = unsafe { &*sym_ptr }; - if !sym.execution_context().is_none() { - return Some(sym.execution_context()); - } - let frame = sym.frame(); - if frame.is_none() { - return None; - } - let ec = ctx.trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[frame], - crate::descr::pyframe_execution_context_descr(), - ); - Some(ec) + (!sym.execution_context().is_none()).then(|| sym.execution_context()) } -/// Eagerly recover the portal EC red before the full-body walk records its -/// first guard, caching it into `sym.execution_context`. +/// Validate the portal EC red before the full-body walk records its first guard. /// /// The portal `[frame, ec]` reds (`interp_jit.py reds = ['frame', 'ec']`) /// are force-alived in every `-live-` op's R-bank, so every guard's resume -/// snapshot lists the EC color. Loop / function-entry syms seed -/// `sym.execution_context = InputArgRef(1)` at `create_sym`, but a -/// bridge-from-guard sym whose ec color collides with a real frame slot is -/// left `OpRef::NONE` by `setup_bridge_sym` (state.rs), which defers -/// the recovery to `ensure_execution_context`. +/// snapshot lists the dedicated EC color. Loop/function-entry syms seed +/// `InputArgRef(1)` and bridge setup decodes the failing guard's corresponding +/// inputarg into the same field. /// /// The walker's snapshot-capture path /// (`walker_capture_snapshot_for_last_guard` → `collect_outer_active_boxes`) @@ -7883,17 +7859,12 @@ fn walker_ensure_execution_context( /// itself is unset, this is a no-op. pub(crate) fn seed_execution_context_for_walk( sym: &mut Sym, - trace_ctx: &mut TraceCtx, + _trace_ctx: &mut TraceCtx, ) { - if !sym.execution_context().is_none() || sym.frame().is_none() { - return; - } - let ec = trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[sym.frame()], - crate::descr::pyframe_execution_context_descr(), + assert!( + sym.frame().is_none() || !sym.execution_context().is_none(), + "full-body walk is missing the portal execution-context red", ); - sym.set_execution_context(ec); } fn walker_int_specialization_operands( diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs index 65f9715b7ff..53cd2d85230 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/residual_call.rs @@ -1319,7 +1319,8 @@ impl ResidualFrameChainGuard { if frame.is_null() { return None; } - let ec = unsafe { (*frame).execution_context } as *mut pyre_interpreter::PyExecutionContext; + let ec = pyre_interpreter::call::getexecutioncontext() + as *mut pyre_interpreter::PyExecutionContext; if ec.is_null() { return None; } @@ -5308,6 +5309,13 @@ pub(crate) fn dispatch_residual_call_iRd_kind( let ei = call_descr.get_extra_info(); repair_carrier_call_ref_args(ctx, op.pc, ei.pyre_helper, &mut r_args); + // Resolve LOAD_GLOBAL's semantic namespace before any specialization reads + // the ref args and before `_build_allboxes` copies them. PyPy's + // `LOAD_GLOBAL` asks the live MIFrame for `get_w_globals()`; keeping that + // one OpRef in both representations prevents the cell fold from seeing the + // codewriter's null trace-time placeholder while the surviving residual + // sees the later replacement. + replace_movable_load_global_namespace_with_frame_globals(ctx, ei.pyre_helper, &mut r_args); // Residual-call entry mirrors `execute_varargs`: even when the walker // folds the call or leaves it recorded symbolically, stale handled // exceptions from earlier opcodes are not visible to the following @@ -5447,8 +5455,7 @@ pub(crate) fn dispatch_residual_call_iRd_kind( // `_r_*` shape: argboxes = R-list only; argbox_types = [Ref; n]. let argbox_types: Vec = vec![Type::Ref; r_args.len()]; - let mut allboxes = build_allboxes(funcptr, &r_args, &argbox_types, call_descr.arg_types()); - replace_movable_load_global_namespace_with_frame_globals(ctx, ei, &mut allboxes); + let allboxes = build_allboxes(funcptr, &r_args, &argbox_types, call_descr.arg_types()); if let Err(e) = ensure_residual_call_args_bound(&allboxes, op.pc) { if fbw_debug_abort_enabled() { let len_pc = op.pc + 1 + 1; @@ -6533,6 +6540,12 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( // otherwise forbid (E0506). let pyre_helper_kind = original_call_descr.get_extra_info().pyre_helper; repair_carrier_call_ref_args(ctx, op.pc, pyre_helper_kind, &mut r_args); + // Do this before the fold block below and before `_build_allboxes`: both + // consumers must observe the same per-MIFrame namespace. In particular, + // the portal codewriter deliberately supplies NULL as a hint placeholder; + // delaying replacement until the generic residual path makes the + // LoadGlobal cell fold decline on that stale value. + replace_movable_load_global_namespace_with_frame_globals(ctx, pyre_helper_kind, &mut r_args); // Void shape `_ir_v/iIRd` (`pyjitpl.py opimpl_residual_call_ir_v = // _opimpl_residual_call2`) has no `>X` dst byte; see // `dispatch_residual_call_iRd_kind` for the void operand-layout note. @@ -7190,8 +7203,6 @@ pub(crate) fn dispatch_residual_call_iIRd_kind( } } - replace_movable_load_global_namespace_with_frame_globals(ctx, ei, &mut allboxes); - // Defer the arg-bound check past the short-circuiting LoadConst / // LoadGlobal folds above: each resolves the call to a constant from // `i_args`/`r_args` without recording it, so an unbound *trailing* arg diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index bc109e42cf6..c8c41e7bfb0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -913,17 +913,16 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( } else { stack_sync }; - // A branch guard (`scope.branch_guard_jitcode_pc`, consumed by the - // stack overlay above for the #124 kept-stack source recovery) keys - // the encoder liveness window on the guard's own jitcode pc — the - // resume `py_pc` is a not-taken merge point whose live colors the - // walk has not written. - // #124 Approach B (M2): carry the guard's raw JitCode byte offset - // as the resume coordinate ONLY for branch guards — they supply - // their own pc in `GuardCaptureScope::branch_guard_jitcode_pc`, the - // kept-stack-across-branch precision `setposition(jitcode, - // miframe.pc)` preserves and the lossy `py_pc → jitcode` - // resume-translation collapses. + // A kept-stack branch guard supplies its own jitcode pc in + // `scope.branch_guard_jitcode_pc` solely as a VALUE-SOURCE + // coordinate for the stack overlay above. It must not become the + // frame's resume coordinate. RPython's + // `MetaInterp.capture_resumedata` temporarily sets `MIFrame.pc` to + // `resumepc` before `get_list_of_active_boxes` and restores the + // tracing pc afterwards: the saved frame therefore resumes at the + // not-taken arm (`op_pc` here), while its already-populated + // register bank still supplies values produced before the branch. + // Keep those two coordinates distinct here as well. // // Every other guard (guard_value / guard_class / guard_no_exception, // the `after_residual_call` family) resumes at a `py_pc` whose @@ -940,11 +939,6 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // decodable `-live-` startpoint by construction — the runtime // resolved it to reach this walk. carried as i32 - } else if let Some(guard_jc_pc) = scope.branch_guard_jitcode_pc { - // The kept-stack branch guard's own `op.pc` (walker - // `MIFrame.pc`) — the ONE carried word not sourced from the - // resume-translation. - guard_jc_pc as i32 } else if !after_residual_call && matches!( ctx.trace_ctx.last_guard_opcode(), @@ -958,11 +952,9 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( { // #366: carry the `-live-` marker offset, NOT the raw guard // `op_pc`. Reached only on the - // default-scope path (`scope.branch_guard_jitcode_pc.is_none()`): the - // specialization guards (`GuardValue`/`GuardClass`) and the - // depth-0 branch guards (`GuardTrue`/`GuardFalse`, kept-stack - // depth>0 branches take the first arm carrying `op.pc`). For - // every guard here the codewrite-time twin resolves the + // specialization guards (`GuardValue`/`GuardClass`) and all + // branch guards (`GuardTrue`/`GuardFalse`). For every guard + // here the codewrite-time twin resolves the // `-live-` marker, keeping the // encoder reg-bank window and decoder liveness symmetric. It is // a valid startpoint (`can_decode_live_vars` holds). @@ -1223,10 +1215,10 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( ), } }; - // A branch guard whose kept operand-stack slot has neither a mirror - // box nor an edge-move entry has no per-slot source; the gate that - // admitted this guard assumed the recovery covered it, so decline - // rather than encode the stale merge-color read. + // A branch guard whose kept operand-stack slot has no mirror box, no + // edge-move entry and no live shadow entry has no per-slot source; + // the gate that admitted this guard assumed a recovery covered it, + // so decline rather than encode the stale merge-color read. // // `get_list_of_active_boxes` (`pyjitpl.py`) has no such case: // it reads `self.registers_r[index]`, and the register bank IS the @@ -2780,17 +2772,18 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( if !callee_pjc.is_populated() || callee_pjc.code_ptr.is_null() { return Err(DispatchError::callee_inline_unsupported(callee_op_pc)); } - // Mirror of the single-frame path: the callee (top) frame carries a branch - // guard's supplied pc (`GuardCaptureScope::branch_guard_jitcode_pc`) - // unchanged. For other guards, the callee payload supplies the - // resume-marker twin: + // Mirror of the single-frame path: the callee (top) frame always carries + // the resume-marker twin for its requested resume coordinate. A branch + // guard's supplied pc (`GuardCaptureScope::branch_guard_jitcode_pc`) is a + // value-source coordinate only; carrying it here would re-run + // `goto_if_not` without its popped condition register. For guards, the + // callee payload supplies the resume-marker twin: // after-residual guards use the fallthrough twin and retain the sentinel // on a miss, while plain guards retain the raw `callee_op_pc` on a miss. // Computed before box collection so encoder liveness and decoder resume // use the same coordinate. - let callee_jitcode_pc: i32 = match scope.branch_guard_jitcode_pc { - Some(g) => g as i32, - None if after_residual_call => after_residual_guard_marker(&callee_pjc, callee_op_pc, None) + let callee_jitcode_pc: i32 = match after_residual_call { + true => after_residual_guard_marker(&callee_pjc, callee_op_pc, None) .or_else(|| { // Fallback only, for the same reason as the single-frame path: // the sticky cursor names this frame's op only while no other @@ -2815,7 +2808,7 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( // The single-frame path can substitute the anchor because it re-reads // the owning frame's vable shadow at the carried coordinate; the callee // sub-walk owns no shadow to re-read. - None => callee_pjc + false => callee_pjc .resume_marker_for_jitcode_pc(callee_op_pc) .map(|m| m as i32) .unwrap_or(callee_op_pc as i32), @@ -2840,21 +2833,19 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( // non-branch captures, its marker construction already applies // the same trivia (and, after a residual call, semantic // fallthrough) transform as the diagnostic inversion above. - let (site, native_marker) = match scope.branch_guard_jitcode_pc { - Some(_) => ("mf_callee_inversion_branch_external", None), - None if after_residual_call => ( + let (site, native_marker) = match after_residual_call { + true => ( "mf_callee_inversion_after_residual", callee_pjc.after_residual_marker_for_jitcode_pc(callee_op_pc), ), - None => ( + false => ( "mf_callee_inversion_plain", callee_pjc.resume_marker_for_jitcode_pc(callee_op_pc), ), }; pcmap_recipe_resultcolor_audit_probe(site, "fire"); - let verdict = match (scope.branch_guard_jitcode_pc, native_marker) { - (Some(_), _) => "branch_external", - (None, Some(marker)) => { + let verdict = match native_marker { + Some(marker) => { let native_py = crate::py_coord::containing_py_pc_for_jitcode_pc( &callee_pjc.metadata, marker, @@ -2865,7 +2856,7 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( "di" } } - (None, None) => "native_miss", + None => "native_miss", }; pcmap_recipe_resultcolor_audit_probe(site, verdict); } diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index 5b0203dab84..8498d4db747 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -9210,15 +9210,13 @@ pub(crate) fn try_walker_specialize_sys_getframe( } } if !inline_level { - // `ec = space.getexecutioncontext()` — recovered off the portal frame, - // the same route `walker_ec_enter` takes (`inline_call.rs`). - let ec_op = ctx.trace_ctx.record_op_with_descr( - OpCode::GetfieldGcR, - &[vable_op], - crate::descr::pyframe_execution_context_descr(), - ); - ctx.trace_ctx - .set_opref_concrete(ec_op, majit_ir::Value::Ref(majit_ir::GcRef(ec as usize))); + // `ec = space.getexecutioncontext()` is the portal's second red, + // carried independently of the virtualizable frame. + let Some(ec_op) = walker_ensure_execution_context(ctx) else { + ctx.trace_ctx.cut_trace_with_snapshots(pre_emit_pos); + ctx.trace_ctx.heap_cache_mut().reset(); + return Ok(None); + }; // `f = ec.gettopframe_nohidden()` followed by // `_do_jit_force_virtual`'s standard-box identity guard. let topframeref_op = ctx.trace_ctx.record_op_with_descr( diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 36955733a78..91202fd80fe 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -2855,7 +2855,7 @@ use crate::descr::{ }; use crate::frame_layout::{ PYFRAME_DEBUGDATA_OFFSET, PYFRAME_LOCALS_CELLS_STACK_OFFSET, PYFRAME_PYCODE_OFFSET, - PYFRAME_VALUESTACKDEPTH_OFFSET, PYFRAME_W_GLOBALS_OFFSET, + PYFRAME_VALUESTACKDEPTH_OFFSET, }; use crate::helpers::emit_box_float_inline; @@ -2873,6 +2873,9 @@ pub use crate::liveness::{LiveVars, liveness_for}; pub struct PyreJitState { #[vable(frame)] pub frame: usize, + /// PyPy portal's second red (`interp_jit.py` `reds = ['frame', 'ec']`). + /// This belongs to the execution-context activation, not to `PyFrame`. + pub execution_context: usize, } /// Meta information for a trace — describes the shape of the code being traced. @@ -3010,8 +3013,9 @@ pub struct PyreSym { pub(crate) vable_valuestackdepth: OpRef, #[vable(inputarg, type = ref)] pub(crate) vable_debugdata: OpRef, - #[vable(inputarg, type = ref)] - pub(crate) vable_w_globals: OpRef, + /// Semantic namespace for this MIFrame. PyPy derives it from the frame's + /// own pycode/debugdata; it is deliberately not a virtualizable scalar. + pub(crate) frame_w_globals: OpRef, #[vable(array_base)] pub(crate) vable_array_base: Option, /// True when this frame's `locals_cells_stack_w` array IS the active @@ -3156,6 +3160,7 @@ pub trait WalkSym { fn vable_array_base(&self) -> Option; fn vable_last_instr(&self) -> OpRef; fn vable_valuestackdepth(&self) -> OpRef; + fn frame_w_globals(&self) -> OpRef; fn last_exc_value(&self) -> pyre_object::PyObjectRef; fn set_last_exc_value(&mut self, value: pyre_object::PyObjectRef); fn last_exc_box(&self) -> OpRef; @@ -3324,6 +3329,11 @@ impl WalkSym for PyreSym { self.vable_valuestackdepth } + #[inline] + fn frame_w_globals(&self) -> OpRef { + self.frame_w_globals + } + #[inline] fn last_exc_value(&self) -> pyre_object::PyObjectRef { self.last_exc_value @@ -3427,7 +3437,7 @@ pub struct TestSymState { pub vable_pycode: OpRef, pub vable_valuestackdepth: OpRef, pub vable_debugdata: OpRef, - pub vable_w_globals: OpRef, + pub frame_w_globals: OpRef, } /// Trace-time view over the virtualizable `PyFrame`. @@ -4879,14 +4889,20 @@ pub(crate) fn trace_float_block_setitem_value( ctx.heapcache_setarrayitem(block, index, descr_idx, value); } -/// pyframe.py `self.w_globals` — read the canonical dict object -/// from the frame. Returns a PyObjectRef (W_DictObject or -/// W_ModuleDictObject). +/// `PyFrame.get_w_globals` common path: read the frame's own pycode, then its +/// first-seen globals. Inline callers only use this after proving that the +/// callee function globals is that code-global identity; the uncommon +/// debugdata override stays on the residual path. pub(crate) fn frame_get_globals_obj(ctx: &mut TraceCtx, frame: OpRef) -> OpRef { - ctx.record_op_with_descr( + let pycode = ctx.record_op_with_descr( OpCode::GetfieldGcR, &[frame], - crate::descr::pyframe_w_globals_obj_descr(), + crate::descr::pyframe_code_descr(), + ); + ctx.record_op_with_descr( + OpCode::GetfieldGcR, + &[pycode], + crate::descr::pycode_w_globals_descr(), ) } @@ -5202,8 +5218,8 @@ impl majit_ir::QuasiImmutHandle for RecordedQuasiImmut { self.0.is_current() } - fn register_loop_token(&self, flag: &std::sync::Arc) { - self.0.register_loop_token(flag); + fn register_loop_token(&self, token: &std::sync::Arc) { + self.0.register_loop_token(token); } fn instance_identity(&self) -> usize { @@ -6572,7 +6588,7 @@ impl PyreSym { vable_pycode: OpRef::NONE, vable_valuestackdepth: OpRef::NONE, vable_debugdata: OpRef::NONE, - vable_w_globals: OpRef::NONE, + frame_w_globals: OpRef::NONE, vable_array_base: None, is_active_vable_owner: false, concrete_locals: Vec::new(), @@ -6693,10 +6709,11 @@ impl PyreSym { /// repopulate the shadow from resume data. `is_active_vable_owner` /// is cleared (`clear_active_vable`) because the bridge's /// inputarg layout lacks the `[frame, last_instr, pycode, - /// valuestackdepth, debugdata, w_globals]` scalar + /// valuestackdepth, debugdata]` scalar /// header that `init_vable_indices` assumes (see /// `virtualizable_spec.rs::PYFRAME_VABLE_FIELDS` for the - /// canonical 5-scalar layout from `interp_jit.py:25-30`); the + /// four concrete scalar fields selected from + /// `interp_jit.py PyFrame._virtualizable_`); the /// frame still owns the shadow /// semantically though. /// @@ -6730,8 +6747,8 @@ impl PyreSym { /// Demote this frame from active virtualizable owner. Used at bridge /// setup (`setup_bridge_sym`) where the bridge's inputarg layout /// does not have the `[frame, last_instr, pycode, valuestackdepth, - /// debugdata, w_globals]` scalar header that the loop-portal - /// `init_vable_indices` assumes (canonical 5-scalar + /// debugdata]` scalar header that the loop-portal + /// `init_vable_indices` assumes (canonical four-scalar /// layout in `virtualizable_spec.rs::PYFRAME_VABLE_FIELDS`); /// subsequent reads consult `bridge_local_oprefs` or fall through /// to the heap array via `locals_cells_stack_array_ref`. @@ -6756,7 +6773,7 @@ impl PyreSym { sym.vable_pycode = state.vable_pycode; sym.vable_valuestackdepth = state.vable_valuestackdepth; sym.vable_debugdata = state.vable_debugdata; - sym.vable_w_globals = state.vable_w_globals; + sym.frame_w_globals = state.frame_w_globals; sym } @@ -6951,8 +6968,12 @@ impl PyreSym { if concrete_frame != 0 { let frame = unsafe { &*(concrete_frame as *const pyre_interpreter::pyframe::PyFrame) }; self.jitcode = jitcode_for(frame.pycode); - self.concrete_namespace = frame.w_globals; - self.concrete_execution_context = frame.execution_context; + self.concrete_namespace = frame.get_w_globals(); + self.frame_w_globals = ctx.const_ref(self.concrete_namespace as usize as i64); + if self.concrete_execution_context.is_null() { + self.concrete_execution_context = + pyre_interpreter::call::getexecutioncontext() as *const _; + } // `PyPyJitDriver.reds = ['frame', 'ec']`: an MIFrame register // contains a Box together with its recording-time value. The // virtualizable seeding below supplies that value for `frame` and @@ -6960,10 +6981,10 @@ impl PyreSym { // virtualizable layout. Stamp it explicitly so blackhole.py's // `_copy_data_from_miframe` analogue can copy the symbolic red // without turning it back into a thread-specific ConstPtr. - if !self.execution_context.is_none() && !frame.execution_context.is_null() { + if !self.execution_context.is_none() && !self.concrete_execution_context.is_null() { ctx.try_set_opref_concrete( self.execution_context, - majit_ir::Value::Ref(majit_ir::GcRef(frame.execution_context as usize)), + majit_ir::Value::Ref(majit_ir::GcRef(self.concrete_execution_context as usize)), ); } self.concrete_vable_ptr = concrete_frame as *mut u8; @@ -7189,10 +7210,7 @@ impl PyreJitState { } fn execution_context_as_usize(&self) -> usize { - let Some(frame_ptr) = self.frame_ptr() else { - return 0; - }; - unsafe { (*(frame_ptr as *const PyFrame)).execution_context as usize } + self.execution_context } fn expanded_virtualizable_live_values_with_extra_reds( @@ -7207,7 +7225,6 @@ impl PyreJitState { self.pycode_as_usize(), self.valuestackdepth(), self.debugdata_as_usize(), - self.w_globals_as_usize(), meta.num_locals, meta.valuestackdepth, |i| self.local_at(i).unwrap_or(PY_NULL) as usize, @@ -7455,9 +7472,8 @@ impl PyreJitState { let Some(frame_ptr) = self.frame_ptr() else { return None; }; - let w_globals = unsafe { - *(frame_ptr.add(PYFRAME_W_GLOBALS_OFFSET) as *const pyre_object::PyObjectRef) - }; + let w_globals = + unsafe { (&*(frame_ptr as *const pyre_interpreter::pyframe::PyFrame)).get_w_globals() }; if w_globals.is_null() { return None; } @@ -7635,34 +7651,18 @@ impl PyreJitState { .expect("PyreJitState.frame must point to a valid PyFrame") } - /// Read the w_globals pointer from the heap frame. - pub fn w_globals_as_usize(&self) -> usize { - self.read_frame_usize(PYFRAME_W_GLOBALS_OFFSET) - .expect("PyreJitState.frame must point to a valid PyFrame") - } - - /// Read the execution context pointer from the heap frame. + /// Read the execution context red independently of the virtualizable frame. /// /// `interp_jit.py reds = ['frame', 'ec']`: ec is a non-vable red - /// inputarg in RPython. pyre's PyFrame carries it inline at - /// `execution_context`, so this accessor derefs the heap; from the - /// macro-generated layout's perspective ec sits at SYM_EC_IDX between - /// the frame pointer and the vable scalar block (`pyjitpl.py:2957 - /// redboxes` then `:2964 + virtualizable_boxes`). + /// inputarg in RPython. It therefore lives on `PyreJitState`, beside the + /// frame red, and never in the `PyFrame` heap layout. pub fn ec_as_usize(&self) -> usize { - self.read_frame_usize(crate::frame_layout::PYFRAME_EXECUTION_CONTEXT_OFFSET) - .expect("PyreJitState.frame must point to a valid PyFrame") + self.execution_context } - /// Write the execution context pointer into the heap frame. - /// - /// Called by `virt_restore_scalars` when reconstructing red inputargs - /// from a guard-failure resume vector. + /// Restore the independent execution-context red from resume data. pub fn set_ec(&mut self, value: usize) { - assert!( - self.write_frame_usize(crate::frame_layout::PYFRAME_EXECUTION_CONTEXT_OFFSET, value), - "PyreJitState.frame must point to a valid PyFrame" - ); + self.execution_context = value; } /// Read the code pointer (pycode) from the heap frame. @@ -7672,7 +7672,12 @@ impl PyreJitState { /// Read the namespace pointer from the heap frame. pub fn namespace_as_usize(&self) -> usize { - self.w_globals_as_usize() + let frame_ptr = self + .frame_ptr() + .expect("PyreJitState.frame must point to a valid PyFrame"); + unsafe { + (&*(frame_ptr as *const pyre_interpreter::pyframe::PyFrame)).get_w_globals() as usize + } } /// Write the pycode pointer to the heap frame. @@ -7684,14 +7689,6 @@ impl PyreJitState { ); } - /// Write the w_globals pointer to the heap frame. - pub fn set_w_globals(&mut self, value: usize) { - assert!( - self.write_frame_usize(PYFRAME_W_GLOBALS_OFFSET, value), - "PyreJitState.frame must point to a valid PyFrame" - ); - } - /// Compatibility wrapper for older callers that still speak in /// terms of `code` / `namespace`. pub fn set_code(&mut self, value: usize) { @@ -7699,7 +7696,13 @@ impl PyreJitState { } pub fn set_namespace(&mut self, value: usize) { - self.set_w_globals(value); + let frame_ptr = self + .frame_ptr() + .expect("PyreJitState.frame must point to a valid PyFrame"); + unsafe { + (&mut *(frame_ptr as *mut pyre_interpreter::pyframe::PyFrame)) + .set_w_globals(value as pyre_object::PyObjectRef); + } } /// pyframe.py:82 debugdata — read from heap frame. @@ -10179,6 +10182,8 @@ impl JitState for PyreJitState { // live frame pointer so the bridge seed sees the real // `locals_cells_stack_w` length. sym.concrete_vable_ptr = self.frame as *mut u8; + sym.concrete_execution_context = + self.execution_context as *const pyre_interpreter::PyExecutionContext; } fn driver_descriptor(&self, _meta: &Self::Meta) -> Option { @@ -10400,7 +10405,7 @@ impl JitState for PyreJitState { .collect(); let bridge_valuestackdepth = concrete_values // virtualizable_values has no ec red: [vable, last_instr, - // pycode, valuestackdepth, debugdata, w_globals, ...]. + // pycode, valuestackdepth, debugdata, ...]. .get(first_vable_scalar_idx + 2) .map(value_to_usize) .unwrap_or(sym.valuestackdepth) @@ -10573,6 +10578,29 @@ impl JitState for PyreJitState { sym.registers_f[reg_idx] = resolved; value_cursor += 1; } + // `interp_jit.py PyPyJitDriver.reds = ['frame', 'ec']`: codewriter gives both + // portal reds dedicated Ref colors and force-keeps them live at every + // guard. `consume_boxes` above therefore rebuilt the bridge's own EC + // inputarg at `portal_ec_reg`; retain that OpRef before converting the + // color-indexed register bank into the semantic locals/stack mirror. + let (_, portal_ec_reg) = portal_red_regs_at(frame0.jitcode_index); + let bridge_execution_context = (portal_ec_reg != u16::MAX) + .then(|| { + bridge_registers_r + .get(portal_ec_reg as usize) + .copied() + .unwrap_or(OpRef::NONE) + }) + .unwrap_or(OpRef::NONE); + assert!( + !bridge_execution_context.is_none(), + "setup_bridge_sym: portal ec red missing at color {} for jitcode {} pc {}; \ + live_refs={:?}", + portal_ec_reg, + frame0.jitcode_index, + frame0.pc, + reg_indices.ref_, + ); // Reconstruct the slot-indexed semantic register file // (`[locals.., stack_tail..]`) from the color-indexed resume decode. // The decode just filled `bridge_registers_r` by abstract-register @@ -10814,8 +10842,8 @@ impl JitState for PyreJitState { // pyre's start_bridge_tracing calls initialize_sym() (which runs // init_symbolic) BEFORE setup_bridge_sym, so init_symbolic sees // bridge_local_oprefs == None and falls into the vable_array_base - // branch (init_vable_indices hard-codes vable_array_base = 7 for - // pyre's 7-slot virtualizable header). That branch produces + // branch (init_vable_indices derives vable_array_base = 6 for + // pyre's `[frame, ec] + four scalars` portal header). That branch produces // OpRef::from_raw(base+i) values from the PARENT trace's namespace, leaving // stale parent OpRefs in registers_r after we set // bridge_local_oprefs here. @@ -10916,29 +10944,21 @@ impl JitState for PyreJitState { types.resize(sym.nlocals, Type::Ref); types }; - // The bridge inputs do NOT have the 7-slot scalar header that + // The bridge inputs do NOT have the six-slot portal header that // init_vable_indices assumes. Demote this frame from active // virtualizable owner so any later LOAD_FAST falling through to // the vable_array_base branch uses the heap-array path instead // of synthesizing parent OpRefs. sym.clear_active_vable(); - // `PyPyJitDriver.reds = ['frame', 'ec']` + // `liveness.compute_liveness`: the explicit `-live-` args keep both // portal reds in every guard's frame-register section. They are not - // semantic PyFrame slots, so the color→slot inversion above correctly - // leaves them out of `sym.registers_r`; restore `ec` separately from - // its dedicated red color, exactly as `rebuild_from_resumedata` fills - // the MIFrame register bank before bridge tracing starts. - let (_, portal_ec_reg) = crate::state::portal_red_regs_at(frame0.jitcode_index); - sym.execution_context = if portal_ec_reg == u16::MAX { - OpRef::NONE - } else { - bridge_registers_r - .get(portal_ec_reg as usize) - .copied() - .filter(|op| !op.is_none()) - .unwrap_or(OpRef::NONE) - }; + // semantic PyFrame slots, so the color->slot inversion above correctly + // leaves them out of `sym.registers_r`. A bridge's inputargs come from + // the failing guard's failargs, so the root trace's historical + // `InputArgRef(1)` cannot be reused; `bridge_execution_context`, read + // off the dedicated red color before the inversion, is the correctly + // renumbered bridge input OpRef. + sym.execution_context = bridge_execution_context; // Both outcomes compile, so only the tally separates the bridge that // carries the live red from the one whose first `ec` consumer re-derives // it off the frame. @@ -10947,7 +10967,7 @@ impl JitState for PyreJitState { } else { crate::trace::fbw_diag::BRIDGE_EC_FROM_PORTAL_RED }); - // pyjitpl.py rebuild_state_after_failure parity: after + // pyjitpl.py `rebuild_state_after_failure` parity: after // a guard failure the tracing-time `virtualizable_boxes` mirror // must be rebuilt from the resume data so subsequent vable // ops see OpRefs drawn from the bridge's inputarg stream, not @@ -10957,7 +10977,7 @@ impl JitState for PyreJitState { // Layout mirrors virtualizable.py read_boxes(): // boxes[0..NUM_SCALARS-1] = scalar fields 1..NUM_SCALARS // (vable_last_instr, vable_pycode, vable_valuestackdepth, - // vable_debugdata, vable_w_globals) + // vable_debugdata) // boxes[NUM_SCALARS-1..NUM_SCALARS-1+array_len] = array items // (bridge_locals followed by reserved stack slots) // boxes[-1] = vable identity (sym.frame) @@ -10979,10 +10999,10 @@ impl JitState for PyreJitState { // setup_bridge_sym time), causing pushes past `nlocals`/the // local prefix to panic at `set_virtualizable_entry_at: index N // out of range for N slots`. A probe captured the - // mismatch directly: root portal sized `vable_boxes_len=25` - // (= 6 + 18 + 1) but a fannkuch bridge fell back to - // `bridge_array_len=14` → `vable_boxes_len=21`, then pushed - // `flat_idx=21` and panicked. Fall back to the metadata-derived + // mismatch directly. With the current four-scalar layout a root whose + // array has 18 slots needs `vable_boxes_len=23` (= 4 + 18 + 1); + // falling back to a shorter live prefix would make a later push run + // past that shadow. Fall back to the metadata-derived // size — `metadata.stack_base + metadata.max_stackdepth` is the // same `nlocals + ncells + max_stackdepth` the codewriter // committed to and `PyFrame::__init__` allocates. @@ -11001,7 +11021,6 @@ impl JitState for PyreJitState { sym.vable_pycode, sym.vable_valuestackdepth, sym.vable_debugdata, - sym.vable_w_globals, ]; // virtualizable.py load_list_of_boxes parity: the OpRef half of // virtualizable_boxes comes from the resume-data stream @@ -11231,12 +11250,7 @@ impl JitState for PyreJitState { .last() .map(|&(_, vref_ptr)| vref_ptr) .unwrap_or(sym.concrete_vable_ptr as usize); - let live_ec = if sym.concrete_vable_ptr.is_null() { - std::ptr::null_mut() - } else { - let frame = sym.concrete_vable_ptr as *const pyre_interpreter::PyFrame; - unsafe { (*frame).execution_context as *mut pyre_interpreter::PyExecutionContext } - }; + let live_ec = sym.concrete_execution_context as *mut pyre_interpreter::PyExecutionContext; if !live_ec.is_null() && restored_top != 0 { let published = unsafe { (*live_ec).topframeref }; // A scope the guard still had open must be republished whatever the @@ -12870,7 +12884,10 @@ mod tests { } fn empty_state() -> PyreJitState { - PyreJitState { frame: 0 } + PyreJitState { + frame: 0, + execution_context: 0, + } } fn compile_function_body(src: &str) -> CodeObject { @@ -13245,6 +13262,7 @@ mod tests { let mut state = empty_state(); state.frame = frame_ptr; + state.execution_context = pyre_interpreter::call::getexecutioncontext() as usize; let meta = state.build_meta(0, &PyreEnv); let with_ec = state.extract_live_values(&meta); @@ -13252,7 +13270,7 @@ mod tests { assert_eq!(with_ec[0], Value::Ref(majit_ir::GcRef(frame_ptr))); assert_eq!( with_ec[1], - Value::Ref(majit_ir::GcRef(frame.execution_context as usize)) + Value::Ref(majit_ir::GcRef(state.execution_context)) ); } @@ -13296,7 +13314,7 @@ mod tests { sym.vable_pycode = code_ref; sym.vable_valuestackdepth = ctx.const_int(3); sym.vable_debugdata = ctx.const_ref(0); - sym.vable_w_globals = namespace_ref; + sym.frame_w_globals = namespace_ref; sym.execution_context = ec_ref; sym.registers_r = vec![local0, stack0, stack1]; sym.symbolic_local_types = vec![Type::Ref]; @@ -13324,7 +13342,6 @@ mod tests { let live_pycode = ctx.const_ref(0x4000); let live_vsd = ctx.const_int(2); let live_debugdata = ctx.const_ref(0x5000); - let live_globals = ctx.const_ref(0x7000); let live_local = ctx.const_ref(0x8000); let live_stack = ctx.const_ref(0x9000); @@ -13339,7 +13356,6 @@ mod tests { (live_pycode, Type::Ref), (live_vsd, Type::Int), (live_debugdata, Type::Ref), - (live_globals, Type::Ref), (live_local, Type::Ref), (live_stack, Type::Ref), (frame_ref, Type::Ref), @@ -13356,7 +13372,6 @@ mod tests { live_pycode, live_vsd, live_debugdata, - live_globals, live_local, live_stack, ] @@ -13390,8 +13405,8 @@ mod tests { assert_eq!(sym.vable_pycode, OpRef::input_arg_ref(3)); assert_eq!(sym.vable_valuestackdepth, OpRef::input_arg_int(4)); assert_eq!(sym.vable_debugdata, OpRef::input_arg_ref(5)); - assert_eq!(sym.vable_w_globals, OpRef::input_arg_ref(6)); - assert_eq!(sym.vable_array_base, Some(7)); + assert!(sym.frame_w_globals.is_none()); + assert_eq!(sym.vable_array_base, Some(6)); assert_eq!(sym.symbolic_local_types.len(), 2); assert_eq!(sym.symbolic_stack_types.len(), 2); } @@ -13419,7 +13434,11 @@ mod tests { frame.fix_array_ptrs(); let frame_ptr = (&mut *frame) as *mut PyFrame as usize; - let mut state = PyreJitState { frame: frame_ptr }; + let execution_context = pyre_interpreter::call::getexecutioncontext() as usize; + let mut state = PyreJitState { + frame: frame_ptr, + execution_context, + }; state.set_next_instr(0); state.set_valuestackdepth(4); let meta = PyreMeta { @@ -13433,17 +13452,16 @@ mod tests { slot_types: vec![Type::Ref, Type::Ref, Type::Ref, Type::Ref], }; let values = vec![ - Value::Ref(GcRef(frame_ptr)), // frame - Value::Ref(GcRef(frame.execution_context as usize)), // ec - Value::Int(8), // last_instr - Value::Ref(GcRef(frame.pycode as usize)), // pycode - Value::Int(4), // valuestackdepth - Value::Ref(GcRef(0)), // debugdata - Value::Ref(GcRef(0)), // w_globals - Value::Ref(GcRef(w_int_new(1) as usize)), // local a - Value::Ref(GcRef(w_int_new(2) as usize)), // local b - Value::Ref(GcRef(w_int_new(3) as usize)), // local c - Value::Int(7), // local i + Value::Ref(GcRef(frame_ptr)), // frame + Value::Ref(GcRef(execution_context)), // ec + Value::Int(8), // last_instr + Value::Ref(GcRef(frame.pycode as usize)), // pycode + Value::Int(4), // valuestackdepth + Value::Ref(GcRef(0)), // debugdata + Value::Ref(GcRef(w_int_new(1) as usize)), // local a + Value::Ref(GcRef(w_int_new(2) as usize)), // local b + Value::Ref(GcRef(w_int_new(3) as usize)), // local c + Value::Int(7), // local i ]; state.restore_expanded_virtualizable_values_with_extra_reds(&meta, &values, 1); @@ -14107,7 +14125,6 @@ mod tests { Type::Ref, // pycode Type::Int, // valuestackdepth Type::Ref, // debugdata - Type::Ref, // w_globals Type::Ref, // local0 Type::Ref, // stale cell Type::Ref, // stack0 @@ -14129,7 +14146,6 @@ mod tests { let local0 = w_int_new(41) as i64; let stale_cell = w_int_new(42) as i64; let stack0 = w_int_new(43) as i64; - let globals = w_int_new(44) as i64; let live_cell = w_int_new(45) as i64; let execution_context = w_int_new(40) as i64; let fail_values = [ @@ -14139,7 +14155,6 @@ mod tests { code_ref as i64, 3, 0, - globals, local0, stale_cell, stack0, @@ -14156,7 +14171,6 @@ mod tests { Type::Ref, Type::Ref, Type::Ref, - Type::Ref, ]; let resume_data = majit_metainterp::ResumeDataResult { frames: vec![RebuiltFrame { @@ -14165,8 +14179,8 @@ mod tests { py_pc: 0, values: vec![ RebuiltValue::Const(majit_ir::Const::Ref(majit_ir::GcRef::NULL)), + RebuiltValue::Box(7, Type::Ref), RebuiltValue::Box(8, Type::Ref), - RebuiltValue::Box(9, Type::Ref), RebuiltValue::Box(0, Type::Ref), RebuiltValue::Box(1, Type::Ref), ], @@ -14183,9 +14197,8 @@ mod tests { RebuiltValue::Box(4, Type::Int), RebuiltValue::Box(5, Type::Ref), RebuiltValue::Box(6, Type::Ref), - RebuiltValue::Box(7, Type::Ref), - RebuiltValue::Box(10, Type::Ref), - RebuiltValue::Box(7, Type::Ref), + RebuiltValue::Box(9, Type::Ref), + RebuiltValue::Box(6, Type::Ref), ], virtualref_values: Vec::new(), storage: None, @@ -14211,9 +14224,9 @@ mod tests { assert_eq!( sym.registers_r, vec![ + OpRef::input_arg_ref(6), OpRef::input_arg_ref(7), OpRef::input_arg_ref(8), - OpRef::input_arg_ref(9), ] ); assert_eq!(sym.symbolic_local_types, vec![Type::Ref, Type::Ref]); @@ -14221,7 +14234,7 @@ mod tests { assert_eq!(sym.execution_context, OpRef::input_arg_ref(1)); assert_eq!( sym.bridge_local_oprefs, - Some(vec![OpRef::input_arg_ref(7), OpRef::input_arg_ref(8)]) + Some(vec![OpRef::input_arg_ref(6), OpRef::input_arg_ref(7)]) ); assert_eq!( sym.concrete_locals, @@ -14231,26 +14244,26 @@ mod tests { assert_eq!( ctx.virtualizable_entry_at(array_base), Some(( - OpRef::input_arg_ref(7), + OpRef::input_arg_ref(6), majit_ir::Value::Ref(majit_ir::GcRef(local0 as usize)), )), ); assert_eq!( ctx.virtualizable_entry_at(array_base + 1), Some(( - OpRef::input_arg_ref(10), + OpRef::input_arg_ref(9), majit_ir::Value::Ref(majit_ir::GcRef(live_cell as usize)), )), ); assert_eq!( ctx.virtualizable_entry_at(array_base + 2), Some(( - OpRef::input_arg_ref(9), + OpRef::input_arg_ref(8), majit_ir::Value::Ref(majit_ir::GcRef(stack0 as usize)), )), ); assert_eq!( - ctx.box_value(OpRef::input_arg_ref(9)), + ctx.box_value(OpRef::input_arg_ref(8)), Some(majit_ir::Value::Ref(majit_ir::GcRef(stack0 as usize))), ); } @@ -14302,7 +14315,6 @@ mod tests { Type::Ref, // pycode Type::Int, // valuestackdepth Type::Ref, // debugdata - Type::Ref, // w_globals Type::Ref, // local0 Type::Ref, // stack0 Type::Ref, // stack1 @@ -14310,7 +14322,7 @@ mod tests { let mut ctx = TraceCtx::for_test_types(&input_types); // The vable static-field types come from `virtualizable_gen.rs`'s - // `inputargs` annotations: int/ref/int/ref/ref. + // `inputargs` annotations: int/ref/int/ref. // Mint typed `OpRef::input_arg_*` variants matching // those tags so variant-aware Eq (`OpRef`'s `PartialEq`) lines up // with what the production `init_vable_indices` produces. @@ -14322,14 +14334,13 @@ mod tests { sym.vable_pycode = OpRef::input_arg_ref(3); sym.vable_valuestackdepth = OpRef::input_arg_int(4); sym.vable_debugdata = OpRef::input_arg_ref(5); - sym.vable_w_globals = OpRef::input_arg_ref(6); // local0 / stack0 / stack1 are Ref-typed per `symbolic_local_types` // / `symbolic_stack_types` below — the macro mints the matching // `InputArgRef` variant. sym.registers_r = vec![ + OpRef::input_arg_ref(6), OpRef::input_arg_ref(7), OpRef::input_arg_ref(8), - OpRef::input_arg_ref(9), ]; sym.symbolic_local_types = vec![Type::Ref]; sym.symbolic_stack_types = vec![Type::Ref, Type::Ref]; @@ -14363,15 +14374,15 @@ mod tests { let jump_args = state.with_ctx(|this, ctx| this.close_loop_args_at(ctx, None, None, 0, None)); - assert_eq!(jump_args.len(), 10); + assert_eq!(jump_args.len(), 9); assert_eq!(jump_args[0], OpRef::input_arg_ref(0)); assert_eq!(jump_args[1], OpRef::input_arg_ref(1)); assert_eq!( - &jump_args[7..], + &jump_args[6..], &[ + OpRef::input_arg_ref(6), OpRef::input_arg_ref(7), - OpRef::input_arg_ref(8), - OpRef::input_arg_ref(9) + OpRef::input_arg_ref(8) ] ); assert_eq!(state.sym().execution_context, OpRef::input_arg_ref(1)); @@ -14402,11 +14413,11 @@ mod tests { let n = crate::virtualizable_gen::NUM_SCALAR_INPUTARGS; let mut input_types = vec![ Type::Ref, // frame + Type::Ref, // ec Type::Int, // next_instr Type::Ref, // pycode Type::Int, // valuestackdepth Type::Ref, // debugdata - Type::Ref, // w_globals ]; input_types.extend(std::iter::repeat(Type::Ref).take(array_len)); let mut ctx = TraceCtx::for_test_types(&input_types); @@ -14418,13 +14429,14 @@ mod tests { // tag, so variant-aware Eq (`OpRef`'s `PartialEq`) requires the // matching variant here too. let mut sym = PyreSym::new_uninit(OpRef::input_arg_ref(0)); + sym.execution_context = OpRef::input_arg_ref(1); sym.nlocals = 1; sym.valuestackdepth = 1; - sym.vable_last_instr = OpRef::input_arg_int(1); - sym.vable_pycode = OpRef::input_arg_ref(2); - sym.vable_valuestackdepth = OpRef::input_arg_int(3); - sym.vable_debugdata = OpRef::input_arg_ref(4); - sym.vable_w_globals = OpRef::input_arg_ref(5); + sym.vable_last_instr = OpRef::input_arg_int(2); + sym.vable_pycode = OpRef::input_arg_ref(3); + sym.vable_valuestackdepth = OpRef::input_arg_int(4); + sym.vable_debugdata = OpRef::input_arg_ref(5); + sym.frame_w_globals = ctx.const_ref(frame.get_w_globals() as usize as i64); sym.registers_r = vec![OpRef::input_arg_ref(6)]; sym.symbolic_local_types = vec![Type::Ref]; sym.symbolic_stack_types = Vec::new(); @@ -14444,10 +14456,9 @@ mod tests { OpRef::input_arg_ref(0), majit_ir::Value::Ref(majit_ir::GcRef(frame_ptr)), &[ - OpRef::input_arg_int(1), - OpRef::input_arg_ref(2), - OpRef::input_arg_int(3), - OpRef::input_arg_ref(4), + OpRef::input_arg_int(2), + OpRef::input_arg_ref(3), + OpRef::input_arg_int(4), OpRef::input_arg_ref(5), ], &[OpRef::input_arg_ref(6)], @@ -14728,6 +14739,7 @@ pub(crate) fn assemble_bridge_inline_pending( .map(|k| concrete_value_from_slot(recipe_slot_to_pyobj(recipe.concrete_r[k]))) .collect(); sym.concrete_namespace = w_globals; + sym.frame_w_globals = ctx.const_ref(w_globals as usize as i64); sym.concrete_execution_context = execution_context; // perform_call threads the caller's `ec` down to every inlined callee // (reds=['frame','ec'], interp_jit.py:67), so the callee shares the @@ -14795,7 +14807,6 @@ pub(crate) fn setup_reconstructed_callee_frame( recipe: &ReconstructRecipe, execution_context: *const pyre_interpreter::PyExecutionContext, ec_box: OpRef, - root_frame_box: OpRef, parent_frames: Vec, ) -> Option<(PendingInlineFrame, Vec)> { let raw_code = recipe.code_ptr as *const pyre_interpreter::CodeObject; @@ -14830,19 +14841,12 @@ pub(crate) fn setup_reconstructed_callee_frame( // loop. // // A bridge whose resume data held no value at the portal `ec` color - // arrives with an empty `ec_box` (`bridge_ec_missing`). Read the red off - // the root frame instead, the way `MIFrame::ensure_execution_context` does - // for the opcode walker: a thread owns one ExecutionContext, so every live - // frame's field names the same object. The constant is left only for a - // root that carries no frame OpRef either. + // arrives with an empty `ec_box` (`bridge_ec_missing`) and falls back to + // the constant. There is no second live source to read it from: `ec` is a + // red in `PyPyJitDriver.reds`, not a `PyFrame` field, so a root frame + // OpRef names an object that does not carry one. let ec_seed = if !ec_box.is_none() { ec_box - } else if !root_frame_box.is_none() { - ctx.record_op_with_descr( - majit_ir::OpCode::GetfieldGcR, - &[root_frame_box], - crate::descr::pyframe_execution_context_descr(), - ) } else { crate::jitcode_dispatch::census_record("ReconstructedCallee::EcConstFallback"); ctx.const_ref(execution_context as i64) @@ -14887,7 +14891,6 @@ pub(crate) fn setup_reconstructed_callee_frame( stack_base, pycode_const, w_globals_const, - ec_seed, ); // `perform_call` (`pyjitpl.py`) is three lines — `newframe` + // `setup_call` + `raise ChangeFrame` — and `newframe` (`:2455-2476`) diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 9235993df8c..ab15f3714c6 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -858,8 +858,8 @@ fn try_commit_midbody_abort_inner( if cf_addr == 0 { return Err(MidBodyDecline::BeforeRun("no live caller frame")); } - let ec = unsafe { (*(cf_addr as *const pyre_interpreter::PyFrame)).execution_context } - as *mut pyre_interpreter::PyExecutionContext; + let ec = + pyre_interpreter::call::getexecutioncontext() as *mut pyre_interpreter::PyExecutionContext; if ec.is_null() { return Err(MidBodyDecline::BeforeRun("null execution context")); } @@ -1812,7 +1812,6 @@ fn drive_bridge_carrier_walk( recipe, root_ec, root_ec_box, - root_frame_box, Vec::new(), ) else { discard_bridge_carrier_walk(ctx, sym, entry_depth, pre_pos, &pre_virtualref_boxes); @@ -2249,7 +2248,6 @@ fn drive_middle_frame_and_thread( middle, root_ec, root_ec_box, - root_frame_box, Vec::new(), ) else { crate::jitcode_dispatch::census_record("P2Drain::MiddleSetupFailed"); @@ -3221,10 +3219,8 @@ fn try_adopt_multi_frame_blackhole( drop(locals_undo); drop(image_ref_roots); drop(root_stack); - let ec = unsafe { - (*(cf_addr as *mut pyre_interpreter::PyFrame)).execution_context - as *mut pyre_interpreter::PyExecutionContext - }; + let ec = + pyre_interpreter::call::getexecutioncontext() as *mut pyre_interpreter::PyExecutionContext; // Rooted for the whole drive: once the tracer stores a `JitVirtualRef` in // the chain the displaced value is a nursery object, and a collection // inside the drive would leave the restore below writing back a pre-move @@ -5671,7 +5667,7 @@ fn loop_inlines_abort_permanent_callee( // the walk mutates anything. unsafe { let cf = &*(cf_addr as *const pyre_interpreter::pyframe::PyFrame); - let root_globals = cf.w_globals; + let root_globals = cf.get_w_globals(); if root_globals.is_null() { return None; } diff --git a/pyre/pyre-jit-trace/src/trace_opcode.rs b/pyre/pyre-jit-trace/src/trace_opcode.rs index adfa87bf9a2..599dd0a9672 100644 --- a/pyre/pyre-jit-trace/src/trace_opcode.rs +++ b/pyre/pyre-jit-trace/src/trace_opcode.rs @@ -175,19 +175,14 @@ pub(crate) extern "C" fn normalize_raise_varargs_jit( exc_obj: i64, cause_obj: i64, ) -> i64 { - let frame_ptr = frame_ptr as *const pyre_interpreter::pyframe::PyFrame; + let _frame_ptr = frame_ptr as *const pyre_interpreter::pyframe::PyFrame; let exc = exc_obj as pyre_object::PyObjectRef; let raw_cause = cause_obj as pyre_object::PyObjectRef; // pyopcode.py:704-722 — cause and exc normalization both run against - // `self.space`/`frame.execution_context`. Pin the caller's frame - // context for the whole body so the cause-class-call and the - // exc-class-call observe the same namespace / thread state. - let frame_ctx = if frame_ptr.is_null() { - std::ptr::null() - } else { - unsafe { (*frame_ptr).execution_context } - }; + // `self.space.getexecutioncontext()`: execution context belongs to the + // current activation/thread, independently of the frame object. + let frame_ctx = pyre_interpreter::call::getexecutioncontext(); let saved_ctx = pyre_interpreter::call::take_last_exec_ctx(); if !frame_ctx.is_null() { pyre_interpreter::call::set_last_exec_ctx(frame_ctx); @@ -993,26 +988,18 @@ impl MIFrame { unsafe { &mut *self.sym } } - pub(crate) fn frame(&self) -> OpRef { - self.sym().frame - } - /// `pypy/module/pypyjit/interp_jit.py reds = ['frame', 'ec']` requires /// every CALL_ASSEMBLER red-args list and JUMP-args list to carry ec. - /// Normal trace setup seeds `sym.execution_context`; this recovery keeps - /// adapter paths from passing OpRef::NONE as the ec red. - pub(crate) fn ensure_execution_context(&mut self, ctx: &mut TraceCtx) -> OpRef { + /// Trace and bridge setup seed `sym.execution_context` from the portal's + /// second red. Adapter paths must preserve that red rather than deriving + /// it from the virtualizable frame. + pub(crate) fn ensure_execution_context(&mut self, _ctx: &mut TraceCtx) -> OpRef { let ec = self.sym().execution_context; - if !ec.is_none() { - return ec; - } - let recovered = ctx.record_op_with_descr( - majit_ir::OpCode::GetfieldGcR, - &[self.frame()], - crate::descr::pyframe_execution_context_descr(), + assert!( + !ec.is_none(), + "MIFrame is missing the portal execution-context red" ); - self.sym_mut().execution_context = recovered; - recovered + ec } #[inline] @@ -1831,7 +1818,7 @@ impl MIFrame { s.vable_last_instr = last_instr_op; s.vable_pycode = pycode_op; s.vable_valuestackdepth = vsd_op; - s.vable_w_globals = w_globals_op; + s.frame_w_globals = w_globals_op; s.owns_virtualizable_shadow() }; // pyjitpl.py `_opimpl_setfield_vable` parity: @@ -1854,12 +1841,6 @@ impl MIFrame { debugdata_op, Value::Ref(GcRef(debugdata)), ); - mirror_vable_static_to_boxes( - ctx, - "w_globals", - w_globals_op, - Value::Ref(GcRef(ns_ptr as usize)), - ); } } @@ -2252,7 +2233,6 @@ impl MIFrame { code, stack_depth, debugdata, - namespace, nlocals, locals, stack, @@ -2335,7 +2315,6 @@ impl MIFrame { ctx.virtualizable_box_at(2) .unwrap_or(s.vable_valuestackdepth), s.vable_debugdata, - s.vable_w_globals, nlocals, locals_vec, stack_vec, @@ -2346,7 +2325,7 @@ impl MIFrame { let mut args = vec![frame]; // NUM_EXTRA_REDS == 1 (crate const-assert): `reds = ['frame', 'ec']`. args.push(execution_context); - args.extend_from_slice(&[next_instr, code, stack_depth, debugdata, namespace]); + args.extend_from_slice(&[next_instr, code, stack_depth, debugdata]); for (idx, value) in locals.into_iter().enumerate() { let target_type = inputarg_types .get(num_scalars + idx) @@ -2530,7 +2509,6 @@ impl MIFrame { 2 => s.vable_pycode = new_opref, 3 => s.vable_valuestackdepth = new_opref, 4 => s.vable_debugdata = new_opref, - 5 => s.vable_w_globals = new_opref, _ => {} }, } @@ -2708,7 +2686,7 @@ impl MIFrame { /// pyjitpl.py capture_resumedata: build fail_args for CURRENT /// top frame. Returns the scalar header plus active_boxes — /// `[frame, (ec)?, last_instr, pycode, valuestackdepth, debugdata, - /// w_globals, active_boxes...]` — matching + /// active_boxes...]` — matching /// `interp_jit.py PyFrame._virtualizable_` / /// `virtualizable_spec.rs::PYFRAME_VABLE_FIELDS` line-by-line. /// `NUM_EXTRA_REDS` controls whether the ec slot @@ -2736,7 +2714,6 @@ impl MIFrame { s.vable_pycode, s.vable_valuestackdepth, s.vable_debugdata, - s.vable_w_globals, ]); fa.extend_from_slice(&active_boxes); fa @@ -3696,6 +3673,7 @@ mod tests { // so their bank-indexed setup is unchanged. sym.nlocals = 2; sym.valuestackdepth = 2; + sym.execution_context = OpRef::input_arg_ref(0); sym.registers_i = vec![OpRef::NONE, OpRef::NONE, int_box]; sym.registers_r = vec![OpRef::NONE, ref_box]; sym.registers_f = vec![OpRef::NONE, OpRef::NONE, OpRef::NONE, float_box]; @@ -3788,6 +3766,7 @@ mod tests { sym.jitcode = inner_jc_ptr; sym.nlocals = 2; sym.valuestackdepth = 3; + sym.execution_context = OpRef::input_arg_ref(0); // Semantic mirror: local0 is at slot 0, while stack depth 0 is at // semantic slot 2. Liveness color 0 belongs to the live stack slot, // reusing dead local0's color. diff --git a/pyre/pyre-jit-trace/src/virtualizable_gen.rs b/pyre/pyre-jit-trace/src/virtualizable_gen.rs index 01de9ddebc0..0f3f04b42ed 100644 --- a/pyre/pyre-jit-trace/src/virtualizable_gen.rs +++ b/pyre/pyre-jit-trace/src/virtualizable_gen.rs @@ -7,7 +7,6 @@ use crate::frame_layout::{ PYFRAME_DEBUGDATA_OFFSET, PYFRAME_LAST_INSTR_OFFSET, PYFRAME_LOCALS_CELLS_STACK_OFFSET, PYFRAME_PYCODE_OFFSET, PYFRAME_VABLE_TOKEN_OFFSET, PYFRAME_VALUESTACKDEPTH_OFFSET, - PYFRAME_W_GLOBALS_OFFSET, }; use crate::state::PyreJitState; use pyre_object::FIXED_OBJECT_ARRAY_TOKEN; @@ -31,16 +30,16 @@ majit_macros::virtualizable! { }, // Layout: [frame:Ref, ec:Ref, last_instr:Int, pycode:Ref, - // valuestackdepth:Int, debugdata:Ref, w_globals:Ref, array...] - // Mirrors `pypy/module/pypyjit/interp_jit.py:25-30`'s - // `_virtualizable_` declaration line by line. `ec` is from - // `interp_jit.py reds = ['frame', 'ec']` (extra_reds above). + // valuestackdepth:Int, debugdata:Ref, array...] + // Mirrors the concrete fields `InstanceRepr._parse_field_list` selects + // from `pypy/module/pypyjit/interp_jit.py`'s `PyFrame._virtualizable_`; + // its stale `w_globals` name has no PyFrame field and is skipped. `ec` is + // from `interp_jit.py reds = ['frame', 'ec']` (extra_reds above). inputargs = { last_instr: Int, pycode: Ref, valuestackdepth: Int, debugdata: Ref, - w_globals: Ref, }, // Array items are PyObjectRef (Ref) @@ -53,7 +52,6 @@ majit_macros::virtualizable! { pycode: ref @ PYFRAME_PYCODE_OFFSET, valuestackdepth: int @ PYFRAME_VALUESTACKDEPTH_OFFSET, debugdata: ref @ PYFRAME_DEBUGDATA_OFFSET, - w_globals: ref @ PYFRAME_W_GLOBALS_OFFSET, }, // RPython virtualizable.py:28 parity: the array field holds a pointer diff --git a/pyre/pyre-jit-trace/src/virtualizable_spec.rs b/pyre/pyre-jit-trace/src/virtualizable_spec.rs index 5cf6e33400a..24ace7bd810 100644 --- a/pyre/pyre-jit-trace/src/virtualizable_spec.rs +++ b/pyre/pyre-jit-trace/src/virtualizable_spec.rs @@ -9,7 +9,10 @@ pub const PYFRAME_VABLE_OWNER_ROOT: &str = "PyFrame"; /// /// This table is the scalar subset of /// `pypy/module/pypyjit/interp_jit.py`'s `_virtualizable_` list, -/// in declaration order. `PyFrame.lastblock` is deliberately absent: +/// in declaration order. RPython's `InstanceRepr._parse_field_list` skips +/// names with no concrete field; after `PyCode.frame_stores_global` removed +/// `PyFrame.w_globals` in upstream commit `bcd8653e5ec`, that stale list entry +/// therefore produces no virtualizable scalar. `PyFrame.lastblock` is deliberately absent: /// the frame model tracked by this tree has no block stack. Unwind uses /// the `co_exceptiontable` lookup at /// `pypy/interpreter/pyopcode.py lookup_exceptiontable`, and pyre's @@ -23,7 +26,6 @@ pub const PYFRAME_VABLE_FIELDS: &[(&str, usize)] = &[ ("pycode", 1), // interp_jit.py:25 pycode ("valuestackdepth", 2), // interp_jit.py:26 valuestackdepth ("debugdata", 3), // interp_jit.py:28 debugdata - ("w_globals", 4), // interp_jit.py:29 w_globals ]; /// Virtualizable array fields in canonical index order. diff --git a/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs b/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs index 9117937af91..0ea059efb3a 100644 --- a/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs +++ b/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs @@ -12,7 +12,10 @@ use pyre_jit_trace::state::PyreJitState; #[test] fn pyre_jit_state_supports_multi_frame_restore() { - let state = PyreJitState { frame: 0 }; + let state = PyreJitState { + frame: 0, + execution_context: 0, + }; assert!( state.supports_multi_frame_restore(), "PyreJitState must override JitState::supports_multi_frame_restore \ diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index c3d180d59fc..174ad04a5e2 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -366,7 +366,10 @@ unsafe fn visit_callee_frame_roots(frame: *mut PyFrame, visitor: &mut dyn FnMut( } visitor(unsafe { &mut *(&mut frame.f_generator_nowref as *mut PyObjectRef as *mut GcRef) }); visitor(unsafe { &mut *(&mut frame.w_yielding_from as *mut PyObjectRef as *mut GcRef) }); - visitor(unsafe { &mut *(&mut frame.w_globals as *mut PyObjectRef as *mut GcRef) }); + if !frame.debugdata.is_null() { + let data = unsafe { &mut *frame.debugdata }; + visitor(unsafe { &mut *(&mut data.w_globals as *mut PyObjectRef as *mut GcRef) }); + } } /// Extra GC root walker for the callee frames a JIT call is running. @@ -1139,12 +1142,10 @@ pub extern "C" fn assembler_call_helper(jitframe_ptr: i64, _virtualizable_ref: i fn resolve_field_offset(owner: &str, field_name: &str) -> usize { use pyre_interpreter::pyframe::PyFrame; match field_name { - "execution_context" => std::mem::offset_of!(PyFrame, execution_context), "code" | "pycode" => std::mem::offset_of!(PyFrame, pycode), "locals_cells_stack_w" => std::mem::offset_of!(PyFrame, locals_cells_stack_w), "valuestackdepth" => std::mem::offset_of!(PyFrame, valuestackdepth), "next_instr" | "f_lasti" | "last_instr" => std::mem::offset_of!(PyFrame, last_instr), - "namespace" | "w_globals" => std::mem::offset_of!(PyFrame, w_globals), "vable_token" => std::mem::offset_of!(PyFrame, vable_token), // #171 codewriter descr-bridge (blackhole side): the dotted nested // `int_items.{len,block}` leaves `_handle_list_call` @@ -1236,22 +1237,27 @@ pub extern "C" fn bh_portal_runner_c( } let frame = unsafe { &mut *frame_ptr }; // warmspot.py:976 calls `portal_ptr(*args)`; it does not copy the green - // `pycode` back onto the red frame. The frame was constructed for this - // activation and already owns its pycode / execution context (the same - // frame-only contract as `ll_portal_runner_shim` above). In particular, + // `pycode` back onto the red frame. The frame owns its pycode while `ec` + // is the independent second red for this activation. In particular, // a stale CRN green must never collapse this frame onto another code // object's identity. if majit_metainterp::majit_log_enabled() - && ((!pycode.is_null() && frame.pycode != pycode as *const ()) - || (!ec.is_null() && frame.execution_context != ec)) + && !pycode.is_null() + && frame.pycode != pycode as *const () { eprintln!( - "[blackhole-resume] CRN/frame identity mismatch: green_pycode={pycode:p} frame_pycode={:p} red_ec={ec:p} frame_ec={:p}", - frame.pycode, frame.execution_context, + "[blackhole-resume] CRN/frame identity mismatch: green_pycode={pycode:p} frame_pycode={:p} red_ec={ec:p}", + frame.pycode, ); } frame.set_last_instr_from_next_instr(next_instr as usize); - match crate::eval::portal_runner_result(frame) { + let saved_ctx = pyre_interpreter::call::take_last_exec_ctx(); + if !ec.is_null() { + pyre_interpreter::call::set_last_exec_ctx(ec); + } + let result = crate::eval::portal_runner_result(frame); + pyre_interpreter::call::set_last_exec_ctx(saved_ctx); + match result { Ok(result) => result as i64, Err(mut err) => { majit_metainterp::blackhole::BH_LAST_EXC_VALUE @@ -1611,8 +1617,8 @@ pub extern "C" fn jit_force_recursive_call_raw_1( /// the callee's code/globals from a function object on every call. The caller /// frame already carries the exact recursive target: /// - `caller.pycode` is the callee code object -/// - `caller.w_globals` is the module globals -/// - `caller.execution_context` is the shared execution context +/// - `caller.get_w_globals()` resolves its code-global/common namespace +/// - the current activation supplies the shared execution context /// /// Trace-time recursive CALL_ASSEMBLER handles the optimized path. The /// concrete helper should mirror RPython's force_fn behavior: execute the @@ -2383,7 +2389,8 @@ fn leave_resumed_blackhole_frame( if frame_ptr.is_null() { return; } - let ec = unsafe { (*frame_ptr).execution_context as *mut pyre_interpreter::PyExecutionContext }; + let ec = + pyre_interpreter::call::getexecutioncontext() as *mut pyre_interpreter::PyExecutionContext; if ec.is_null() { return; } @@ -3218,15 +3225,15 @@ fn handle_blackhole_result(bh_result: BlackholeResult, _green_key: u64) -> Optio // The red frame is the activation identity. PyPy forwards the // CRN arguments to `portal_ptr` without mutating that frame; pyre's // frame-only portal entry likewise reads the pycode / execution - // context already stored on the callee frame. Do not overwrite - // them from a possibly stale green snapshot. + // context is the separate red argument. Do not overwrite pycode + // from a possibly stale green snapshot. if majit_metainterp::majit_log_enabled() - && ((!pycode.is_null() && frame.pycode != pycode as *const ()) - || (!ec.is_null() && frame.execution_context != ec)) + && !pycode.is_null() + && frame.pycode != pycode as *const () { eprintln!( - "[blackhole-resume] CALL_ASSEMBLER CRN/frame identity mismatch: green_pycode={pycode:p} frame_pycode={:p} red_ec={ec:p} frame_ec={:p}", - frame.pycode, frame.execution_context, + "[blackhole-resume] CALL_ASSEMBLER CRN/frame identity mismatch: green_pycode={pycode:p} frame_pycode={:p} red_ec={ec:p}", + frame.pycode, ); } frame.set_last_instr_from_next_instr(next_instr); @@ -3236,7 +3243,13 @@ fn handle_blackhole_result(bh_result: BlackholeResult, _green_key: u64) -> Optio // at its peak stack use. Re-derive the depth from the resume pc — // the CALL_ASSEMBLER-path mirror of the eval.rs CRN handoff. crate::eval::correct_resume_vsd(frame, next_instr); - match crate::eval::portal_runner_result(frame) { + let saved_ctx = pyre_interpreter::call::take_last_exec_ctx(); + if !ec.is_null() { + pyre_interpreter::call::set_last_exec_ctx(ec); + } + let result = crate::eval::portal_runner_result(frame); + pyre_interpreter::call::set_last_exec_ctx(saved_ctx); + match result { Ok(result) => Some(result as i64), Err(mut err) => { let exc_obj = err.to_exc_object(); @@ -3423,7 +3436,7 @@ pub fn trace_and_compile_from_bridge( { let (driver, _) = crate::eval::driver_pair(); - driver.clear_last_compiled_artifact_invalidation_flag(); + driver.clear_last_compiled_artifact_token(); } use crate::eval::build_jit_state; @@ -4591,18 +4604,22 @@ fn fill_positional_defaults_for_jit_call<'a>( } fn create_callee_frame_impl_1_boxed( - caller_frame: i64, + _caller_frame: i64, callable: PyObjectRef, boxed_arg: PyObjectRef, ) -> i64 { let w_code = unsafe { pyre_interpreter::getcode(callable) }; - let caller = unsafe { &*(caller_frame as *const PyFrame) }; let w_globals = unsafe { function_get_globals_obj(callable) }; let one_arg = [boxed_arg]; let args = fill_positional_defaults_for_jit_call(callable, w_code, &one_arg); let args = args.as_ref(); - alloc_callee_frame(w_code, args, w_globals, caller.execution_context) as i64 + alloc_callee_frame( + w_code, + args, + w_globals, + pyre_interpreter::call::getexecutioncontext(), + ) as i64 } fn create_self_recursive_callee_frame_impl_1_boxed( @@ -4611,8 +4628,8 @@ fn create_self_recursive_callee_frame_impl_1_boxed( ) -> i64 { let caller = unsafe { &*(caller_frame as *const PyFrame) }; let func_code = caller.pycode; - let w_globals = caller.w_globals; - let execution_context = caller.execution_context; + let w_globals = caller.get_w_globals(); + let execution_context = pyre_interpreter::call::getexecutioncontext(); // Read before the call: `alloc_callee_frame` resolves `__builtins__`, which // can run a user `__getitem__` and so collect. What this line reports is @@ -4629,9 +4646,12 @@ fn create_self_recursive_callee_frame_impl_1_boxed( frame_ptr as i64 } -fn create_callee_frame_impl(caller_frame: i64, callable: i64, args: &[PyObjectRef]) -> i64 { - let caller = unsafe { &*(caller_frame as *const PyFrame) }; - create_callee_frame_in_ctx(caller.execution_context, callable as PyObjectRef, args) +fn create_callee_frame_impl(_caller_frame: i64, callable: i64, args: &[PyObjectRef]) -> i64 { + create_callee_frame_in_ctx( + pyre_interpreter::call::getexecutioncontext(), + callable as PyObjectRef, + args, + ) } /// [`create_callee_frame_impl`] with the execution context passed directly. @@ -4699,8 +4719,8 @@ pub extern "C" fn jit_create_self_recursive_callee_frame_1_raw_int( ) -> i64 { let caller = unsafe { &*(caller_frame as *const PyFrame) }; let func_code = caller.pycode; - let w_globals = caller.w_globals; - let execution_context = caller.execution_context; + let w_globals = caller.get_w_globals(); + let execution_context = pyre_interpreter::call::getexecutioncontext(); let boxed = pyre_object::intobject::w_int_new(raw_int_arg); @@ -6536,8 +6556,8 @@ pub extern "C" fn bh_box_int_fn(value: i64) -> i64 { /// `(frame: Ref, exc: Ref, cause: Ref) → Ref`. The frame pointer is /// emitted explicitly by `Instruction::RaiseVarargs` (codewriter.rs) /// via `portal_frame_reg`, mirroring `bh_load_global_fn`'s frame-as-arg -/// ABI. `pyopcode.py RAISE_VARARGS` runs inside an opcode -/// dispatch where `frame` and `frame.execution_context` are always +/// ABI. `pyopcode.py RAISE_VARARGS` runs inside an opcode +/// dispatch where `frame` and the current execution context are always /// valid, so `frame_ptr == 0` here signals a wiring bug — fail fast /// at entry rather than degrade silently to a `RuntimeError`. pub extern "C" fn bh_normalize_raise_varargs_with_frame( @@ -6556,10 +6576,10 @@ pub extern "C" fn bh_normalize_raise_varargs_with_frame( let raw_cause = cause as PyObjectRef; // pyopcode.py:704-722 — cause and exc normalization share - // `self.space` / `frame.execution_context`. Pin the caller frame's - // execution_context for the whole body so the cause-class-call and - // exc-class-call observe the same namespace. - let frame_ctx = unsafe { (*parent_frame_ptr).execution_context }; + // `self.space.getexecutioncontext()`. Pin the current activation for the + // whole body so the cause-class-call and exc-class-call observe the same + // thread state. + let frame_ctx = pyre_interpreter::call::getexecutioncontext(); let saved_ctx = pyre_interpreter::call::take_last_exec_ctx(); if !frame_ctx.is_null() { pyre_interpreter::call::set_last_exec_ctx(frame_ctx); @@ -6598,9 +6618,8 @@ pub extern "C" fn bh_normalize_raise_varargs_with_frame( if pyre_object::is_exception(exc) { exc } else if pyre_interpreter::baseobjspace::exception_is_valid_obj_as_class_w(exc) { - // pyopcode.py — `space.call_function(w_type)` does - // not depend on `frame.execution_context`; if the field is - // null on a valid frame the class-call still proceeds. + // pyopcode.py RAISE_VARARGS — `space.call_function(w_type)` does + // not depend on frame-owned context state. // // It runs the exception's `__init__`. `exc` is still unrooted and // the cause normalized above is a fresh nursery object, so both diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index d7fc53972e9..57f201ecd43 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -1430,7 +1430,6 @@ unsafe fn pyframe_object_custom_trace(obj_addr: usize, f: &mut dyn FnMut(*mut ma f(&mut frame.w_yielding_from as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); f(&mut frame.w_builtin as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); - f(&mut frame.w_globals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); if !frame.debugdata.is_null() { let debugdata = frame.debugdata; @@ -1451,6 +1450,7 @@ unsafe fn pyframe_object_custom_trace(obj_addr: usize, f: &mut dyn FnMut(*mut ma // phase-agnostic jitframe.py `jitframe_trace` contract. if walk_fields { let d = unsafe { &mut *frame.debugdata }; + f(&mut d.w_globals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); f(&mut d.w_locals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); f(&mut d.w_extra_locals as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); f(&mut d.w_f_trace as *mut pyre_object::PyObjectRef as *mut majit_ir::GcRef); @@ -3252,12 +3252,13 @@ fn build_gc() -> Box { vec![majit_rlib::rbigint::RBIGINT_DIGITS_OFFSET], )); pyre_object::longobject::set_bigint_gc_type_id(bigint_tid); - // PyPy's FrameDebugData is a plain GC object. It owns three PyObjectRef - // fields; once the frame custom trace greys the payload, the ordinary - // offset walker finds all of them during a major mark. + // PyPy's FrameDebugData is a plain GC object. Once the frame custom trace + // greys a nursery payload, the ordinary offset walker must find every + // PyObjectRef field, including the uncommon per-frame globals override. let frame_debug_data_tid = gc.register_type(TypeInfo::with_gc_ptrs( std::mem::size_of::(), vec![ + std::mem::offset_of!(pyre_interpreter::pyframe::FrameDebugData, w_globals), std::mem::offset_of!(pyre_interpreter::pyframe::FrameDebugData, w_locals), std::mem::offset_of!(pyre_interpreter::pyframe::FrameDebugData, w_extra_locals), std::mem::offset_of!(pyre_interpreter::pyframe::FrameDebugData, w_f_trace), @@ -4803,7 +4804,6 @@ fn install_pyre_object_hooks() { const T: u32 = pyre_interpreter::pyframe::PYFRAME_GC_TYPE_ID; majit_gc::bh_probe_set_field_names(&[ (T, 0, "ob_type"), - (T, fl::PYFRAME_EXECUTION_CONTEXT_OFFSET, "execution_context"), (T, fl::PYFRAME_PYCODE_OFFSET, "pycode"), ( T, @@ -4829,7 +4829,6 @@ fn install_pyre_object_hooks() { (T, fl::PYFRAME_W_YIELDING_FROM_OFFSET, "w_yielding_from"), (T, fl::PYFRAME_F_BACKREF_OFFSET, "f_backref"), (T, fl::PYFRAME_W_BUILTIN_OFFSET, "w_builtin"), - (T, fl::PYFRAME_W_GLOBALS_OFFSET, "w_globals"), ]); majit_gc::bh_probe_set_type_namer(|addr| { let obj = addr as pyre_object::PyObjectRef; @@ -5779,8 +5778,7 @@ impl __extend__ { /// except ExitFrame: ... /// /// In pyre, the JIT-instrumented dispatch loop is eval_loop_jit(). - /// pycode and ec are stored on the frame; eval_loop_jit reads them - /// from frame.pycode and frame.execution_context respectively. + /// `pycode` belongs to the frame while `ec` is the separate portal red. pub fn dispatch( frame: &mut PyFrame, _pycode: pyre_object::PyObjectRef, @@ -5921,13 +5919,15 @@ pub fn _call_not_in_trace( #[inline] fn green_key_from_pycode(next_instr: usize, w_pycode: pyre_object::PyObjectRef) -> Option { - // Safety: this follows existing wrappers that treat `PyCode` - // as an owned pointer to a `CodeObject`. - let code_ptr = unsafe { pyre_interpreter::pycode::w_code_get_ptr(w_pycode) }; - if code_ptr.is_null() { + // `PyFrame.pycode`, `interp_jit.PyPyJitDriver.greens`, and + // `JitCell.comparekey` all carry the PyCode object itself. Unwrapping it + // to the host-side CodeObject creates a second warm cell for the same + // Python function: function-entry dispatch uses `frame.pycode`, while + // marker dispatch would otherwise use `PyCode.code_ptr`. + if w_pycode.is_null() { return None; } - Some(make_green_key(code_ptr, next_instr)) + Some(make_green_key(w_pycode.cast(), next_instr)) } /// The typed `GreenKey` behind [`green_key_from_pycode`]'s `u64`. @@ -5948,16 +5948,15 @@ fn green_key_typed_from_pycode( next_instr: usize, w_pycode: pyre_object::PyObjectRef, ) -> Option { - // Safety: as `green_key_from_pycode` — `PyCode` is an owned pointer to a - // `CodeObject`. - let code_ptr = unsafe { pyre_interpreter::pycode::w_code_get_ptr(w_pycode) }; - if code_ptr.is_null() { + // Keep the compare key on the same PyCode identity as + // `green_key_from_pycode`; the typed and hash paths must name one cell. + if w_pycode.is_null() { return None; } Some(majit_ir::pypyjit_greenkey( next_instr, false, - code_ptr as u64, + w_pycode as u64, )) } @@ -6442,19 +6441,22 @@ pub fn make_green_key(code_ptr: *const (), pc: usize) -> u64 { /// that window reports the loop invalid instead of accepting a registration /// nothing will ever sweep again. /// -/// Lives here rather than in `record_loop_or_bridge` only because the -/// invalidation flag is the compiling driver's, not the metainterp's. +/// Lives here rather than in `record_loop_or_bridge` only because this layer +/// owns the concrete interpreter-side `QuasiImmut` registration call. The +/// metainterp still supplies the exact `original_jitcell_token` that upstream +/// places behind `wref`, including for an attached bridge. pub(crate) fn register_quasi_immutable_deps(_green_key: u64) { let (driver, _) = driver_pair(); let deps = std::mem::take(&mut driver.meta_interp_mut().last_quasi_immutable_deps); if deps.is_empty() { return; } - let Some(flag) = driver.last_compiled_artifact_invalidation_flag() else { + let Some(token) = driver.last_compiled_artifact_token() else { return; }; + let token: std::sync::Arc = token; for qmut in deps { - qmut.register_loop_token(&flag); + qmut.register_loop_token(&token); } } @@ -8398,20 +8400,25 @@ pub(crate) fn pyre_portal_runner( } let frame = unsafe { &mut *frame_ptr }; // warmspot.py:976 forwards the CRN values to `portal_ptr`; it does not - // rewrite the red frame's identity. Pyre's portal dispatch is frame-only, - // so retain the activation's own pycode / execution context rather than - // replacing them from a potentially stale green snapshot. + // rewrite the red frame's identity. Retain the activation's own pycode + // and carry `ec` independently as the second red. if majit_metainterp::majit_log_enabled() - && ((!pycode.is_null() && frame.pycode != pycode as *const ()) - || (!ec.is_null() && frame.execution_context != ec)) + && !pycode.is_null() + && frame.pycode != pycode as *const () { eprintln!( - "[blackhole-resume] portal CRN/frame identity mismatch: green_pycode={pycode:p} frame_pycode={:p} red_ec={ec:p} frame_ec={:p}", - frame.pycode, frame.execution_context, + "[blackhole-resume] portal CRN/frame identity mismatch: green_pycode={pycode:p} frame_pycode={:p} red_ec={ec:p}", + frame.pycode, ); } frame.set_last_instr_from_next_instr(next_instr); - match portal_runner_result(frame) { + let saved_ctx = pyre_interpreter::call::take_last_exec_ctx(); + if !ec.is_null() { + pyre_interpreter::call::set_last_exec_ctx(ec); + } + let result = portal_runner_result(frame); + pyre_interpreter::call::set_last_exec_ctx(saved_ctx); + match result { Ok(result) => Ok((BhReturnType::Ref, result as i64)), Err(mut err) => Err(JitException::ExitFrameWithExceptionRef(majit_ir::GcRef( err.to_exc_object() as usize, @@ -8794,7 +8801,7 @@ fn eval_loop_jit(frame: &mut PyFrame) -> LoopResult { // untranslated body is a no-op; source translation // recognizes this method call and lowers it to JitCode // `jit_merge_point` rather than leaving a residual call. - let marker_ec = unsafe { &*f }.execution_context as *const PyExecutionContext; + let marker_ec = pyre_interpreter::call::getexecutioncontext(); let marker_pycode = unsafe { &*f }.pycode as pyre_object::PyObjectRef; let marker_profiled = unsafe { &*f }.get_is_being_profiled(); pypyjitdriver.jit_merge_point( @@ -8837,7 +8844,7 @@ fn eval_loop_jit(frame: &mut PyFrame) -> LoopResult { // PyPy's `actionflag.decrement_ticker(decr_by)` invariant); // the `action_dispatcher` slow path itself is still a stub // pending the actionflag port. - let ec_ptr = unsafe { &*f }.execution_context as *mut PyExecutionContext; + let ec_ptr = pyre_interpreter::call::getexecutioncontext() as *mut PyExecutionContext; if !ec_ptr.is_null() { // Keep the JIT portal's concrete dispatch in lockstep with // `pyre_interpreter::eval::eval_loop`'s opcode boundary. The @@ -8975,7 +8982,7 @@ fn eval_loop_jit(frame: &mut PyFrame) -> LoopResult { let f: *mut PyFrame = frame_root.frame() as *mut PyFrame; // ── can_enter_jit (RPython interp_jit.py:114) ── // RPython interp_jit.py:114 → warmstate.py:446 - let marker_ec = unsafe { &*f }.execution_context as *const PyExecutionContext; + let marker_ec = pyre_interpreter::call::getexecutioncontext(); let marker_pycode = unsafe { &*f }.pycode as pyre_object::PyObjectRef; let marker_profiled = unsafe { &*f }.get_is_being_profiled(); pypyjitdriver.can_enter_jit( @@ -9970,7 +9977,7 @@ fn execute_assembler( // died. Bracket the assembler run the way `install_current_frame` / // `CurrentFrameGuard` bracket a frame: a balanced run restores the same // value it saved, an unbalanced exit restores the caller. - let ec_for_topframeref = frame_root.frame().execution_context as *mut PyExecutionContext; + let ec_for_topframeref = jit_state.execution_context as *mut PyExecutionContext; // warmstate.py:395 func_execute_token(loop_token, *args) → deadframe let outcome = { let _topframeref_guard = TopFrameRefGuard::new(ec_for_topframeref); @@ -10637,8 +10644,25 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { } } } - let green_key = make_green_key(frame_root.frame().pycode, frame_root.frame().next_instr()); + let function_entry_pc = frame_root.frame().next_instr(); + let function_entry_w_pycode = frame_root.frame().pycode; + let green_key_hash = make_green_key(function_entry_w_pycode, function_entry_pc); let (driver, info) = driver_pair(); + // `warmstate.py maybe_compile_and_run` resolves the full green tuple to one + // JitCell and then carries that cell's procedure token to the runner. Pyre + // carries a cell key instead. The common unchained case returns the bucket + // hash without constructing a GreenKey; only a chained bucket builds the + // typed tuple needed to select its matching cell. + let green_key = driver.meta_interp().resolve_cell_key( + green_key_hash, + Some(&|| { + green_key_typed_from_pycode( + function_entry_pc, + function_entry_w_pycode as pyre_object::PyObjectRef, + ) + .expect("nonnull function-entry PyCode has a typed green key") + }), + ); // RPython warmstate.py maybe_compile_and_run fast path: // if no runnable compiled loop and not tracing, just tick the counter. @@ -10646,10 +10670,13 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { // `compiled_loops` meta) is treated as not-yet-runnable so the counter // keeps ticking toward compiling the real loop. if !driver.has_runnable_compiled_loop(green_key) && !driver.is_tracing() { - let should_trace = driver - .meta_interp_mut() - .warm_state_mut() - .should_trace_function_entry(green_key); + let should_trace = driver.meta_interp_mut().should_trace_function_entry( + green_key_hash, + ( + frame_root.frame().pycode as usize, + frame_root.frame().next_instr(), + ), + ); if !should_trace { return None; } @@ -10689,7 +10716,7 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { // the actions at a bytecode boundary and delivers what they raise // through `handle_exception`. if pyre_interpreter::module::thread::gil::threads_initialized() { - let ec = frame_root.frame().execution_context as *mut PyExecutionContext; + let ec = pyre_interpreter::call::getexecutioncontext() as *mut PyExecutionContext; if !ec.is_null() { let ticker = unsafe { (*ec).actionflag.decrement_ticker( @@ -10931,7 +10958,7 @@ pub fn try_function_entry_jit(frame: &mut PyFrame) -> Option { let next_instr = frame_root.frame().next_instr(); match compile_and_run_once( frame_root.frame(), - green_key, + green_key_hash, next_instr, CompileOnceStart::FunctionEntry, driver, @@ -12747,12 +12774,10 @@ fn build_resumed_frames( let ni_idx = pyre_jit_trace::virtualizable_gen::SYM_LAST_INSTR_IDX as usize - extra; let code_idx = pyre_jit_trace::virtualizable_gen::SYM_PYCODE_IDX as usize - extra; let vsd_idx = pyre_jit_trace::virtualizable_gen::SYM_VALUESTACKDEPTH_IDX as usize - extra; - let ns_idx = pyre_jit_trace::virtualizable_gen::SYM_W_GLOBALS_IDX as usize - extra; // Resolve ALL vable fields from resume data. // vable_values = [frame_ptr(0), last_instr(1), pycode(2), - // valuestackdepth(3), debugdata(4), - // w_globals(5), array...] + // valuestackdepth(3), debugdata(4), array...] // RPython reader.load_next_value_of_type reads ALL values sequentially. let resolved_vable: Vec = (0..vable_values.len()) .map(|i| { @@ -12805,16 +12830,7 @@ fn build_resumed_frames( }) .unwrap_or(std::ptr::null()); - let vable_ns: *const () = resolved_vable - .get(ns_idx) - .map(|v| match v { - Value::Ref(r) => r.as_usize() as *const (), - Value::Int(v) => *v as *const (), - _ => std::ptr::null(), - }) - .unwrap_or(std::ptr::null()); - - // pyjitpl.py synchronize_virtualizable on guard-failure + // pyjitpl.py `synchronize_virtualizable` on guard-failure // bridge entry: stores `self.virtualizable_boxes`, resets the token, // then calls `self.synchronize_virtualizable()` which ends at // virtualizable.py `write_boxes`. `ResumeVableMode::GuardFailureSync` @@ -12849,7 +12865,7 @@ fn build_resumed_frames( f.next_instr(), f.valuestackdepth, f.pycode, - f.w_globals, + f.get_w_globals(), f.debugdata, f.vable_token, locals_w!(f).len(), @@ -12925,12 +12941,11 @@ fn build_resumed_frames( } else { values.len() }; - // virtualizable.py:86-99: namespace from resume data. + // PyFrame.get_w_globals: namespace comes from this frame's restored + // pycode/debugdata pair; it is not a physical virtualizable scalar. let namespace = if is_outermost { - if !vable_ns.is_null() { - vable_ns - } else if !vable_frame_ptr.is_null() { - unsafe { (*vable_frame_ptr).w_globals as *const () } + if !vable_frame_ptr.is_null() { + unsafe { (*vable_frame_ptr).get_w_globals() as *const () } } else { std::ptr::null() } @@ -12947,10 +12962,10 @@ fn build_resumed_frames( if !callee_ns.is_null() { callee_ns } else { - vable_ns + std::ptr::null() } } else { - vable_ns + std::ptr::null() }; result.push(crate::call_jit::ResumedFrame { code: w_code, @@ -13160,6 +13175,7 @@ pub(crate) fn build_jit_state( ) -> PyreJitState { let mut jit_state = PyreJitState { frame: frame as *const PyFrame as usize, + execution_context: pyre_interpreter::call::getexecutioncontext() as usize, }; assert!( jit_state.sync_from_virtualizable(virtualizable_info), @@ -13659,6 +13675,24 @@ impl majit_metainterp::resume::BlackholeAllocator for PyreBlackholeAllocator { mod tests { use super::*; + /// `interp_jit.PyPyJitDriver.greens` carries the PyCode object, not the + /// host CodeObject hidden behind pyre's wrapper. The hash-only marker + /// path and typed `JitCell.comparekey` path must therefore agree with + /// function-entry dispatch even when the wrapper's internal pointer is + /// absent (the supported test-stub representation). + #[test] + fn pycode_green_keys_preserve_wrapper_identity() { + let w_code = pyre_interpreter::pycode::w_code_new(std::ptr::null()); + assert!(!w_code.is_null()); + + let hash = green_key_from_pycode(17, w_code).expect("nonnull PyCode has a green key"); + let typed = + green_key_typed_from_pycode(17, w_code).expect("nonnull PyCode has a typed green key"); + + assert_eq!(hash, make_green_key(w_code.cast(), 17)); + assert_eq!(typed.get_uhash(), hash); + } + /// Read a global by name from the frame's canonical `w_globals` object. #[allow(dead_code)] fn frame_global(frame: &PyFrame, name: &str) -> pyre_object::PyObjectRef { @@ -14653,12 +14687,12 @@ mod tests { symbolic_stack_types: vec![], registers_r: vec![local], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(resume_pc as i64 - 1), vable_pycode: ctx.const_ref(frame.pycode as usize as i64), vable_valuestackdepth: ctx.const_int(1), vable_debugdata: ctx.const_ref(frame.debugdata as usize as i64), - vable_w_globals: ctx.const_ref(frame.w_globals as usize as i64), + frame_w_globals: ctx.const_ref(frame.get_w_globals() as usize as i64), } } @@ -14738,7 +14772,11 @@ mod tests { "frame-value count must come from the same compiled jitcode liveness block" ); - let mut state = PyreJitState { frame: frame_ptr }; + let ec_value = pyre_interpreter::call::getexecutioncontext() as usize; + let mut state = PyreJitState { + frame: frame_ptr, + execution_context: ec_value, + }; state.set_next_instr(0); state.set_valuestackdepth(4); let meta = PyreMeta { @@ -14754,15 +14792,14 @@ mod tests { slot_types: vec![Type::Ref, Type::Ref, Type::Ref, Type::Ref], }; - let ec_value = unsafe { (*(frame_ptr as *const PyFrame)).execution_context as usize }; let mut values = vec![ - Value::Ref(GcRef(frame_ptr)), // frame - Value::Ref(GcRef(ec_value)), // ec extra red - Value::Int(8), // last_instr - Value::Ref(GcRef(frame.pycode as usize)), // pycode - Value::Int(4), // valuestackdepth - Value::Ref(GcRef(0)), // debugdata - Value::Ref(GcRef(frame.w_globals as usize)), // w_globals + Value::Ref(GcRef(frame_ptr)), // frame + Value::Ref(GcRef(ec_value)), // ec extra red + Value::Int(8), // last_instr + Value::Ref(GcRef(frame.pycode as usize)), // pycode + Value::Int(4), // valuestackdepth + Value::Ref(GcRef(0)), // debugdata + Value::Ref(GcRef(frame.get_w_globals() as usize)), // w_globals ]; for reg in live_regs.iter() { match *reg { @@ -14849,14 +14886,14 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Int], registers_r: vec![OpRef::NONE; max_color + 1], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(999), vable_pycode: ctx.const_ref(0xdead), vable_valuestackdepth: ctx.const_int(111), vable_debugdata: ctx.const_ref(0xbeef), - vable_w_globals: ctx.const_ref(0xfeed), + frame_w_globals: ctx.const_ref(0xfeed), }); - let ec_ref = ctx.const_ref(frame.execution_context as usize as i64); + let ec_ref = ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); sym.set_test_execution_context(ec_ref); let mut state = MIFrame::from_sym(&mut ctx, &mut sym, frame_ptr, resume_pc, resume_pc); state.set_resume_marker_for_test(resume_pc); @@ -14883,8 +14920,10 @@ mod tests { ctx.constants_get_value(fail_args[4]), Some(majit_ir::Value::Int(4)), ); - // pycode / debugdata / w_globals stay bound to the trace-start - // inputarg OpRefs the fixture seeded above. + // pycode / debugdata stay bound to the trace-start inputarg OpRefs + // the fixture seeded above. `w_globals` is not a virtualizable + // scalar: PyPy's `PyFrame.get_w_globals` derives it from the live + // frame's pycode/debugdata pair. assert_eq!( ctx.constants_get_value(fail_args[3]), Some(majit_ir::Value::Ref(majit_ir::GcRef(0xdead))), @@ -14893,10 +14932,6 @@ mod tests { ctx.constants_get_value(fail_args[5]), Some(majit_ir::Value::Ref(majit_ir::GcRef(0xbeef))), ); - assert_eq!( - ctx.constants_get_value(fail_args[6]), - Some(majit_ir::Value::Ref(majit_ir::GcRef(0xfeed))), - ); } #[test] @@ -14957,14 +14992,14 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Int], registers_r, concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(0), vable_pycode: ctx.const_ref(0), vable_valuestackdepth: ctx.const_int(0), vable_debugdata: ctx.const_ref(0), - vable_w_globals: ctx.const_ref(0), + frame_w_globals: ctx.const_ref(0), }); - let ec_ref = ctx.const_ref(frame.execution_context as usize as i64); + let ec_ref = ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); sym.set_test_execution_context(ec_ref); trace_state::seed_compiled_trace_jitcode_test_state( &mut sym, @@ -15052,13 +15087,15 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Int], registers_r: vec![OpRef::NONE; max_color + 1], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(0), vable_pycode: ctx.const_ref(0), vable_valuestackdepth: ctx.const_int(0), vable_debugdata: ctx.const_ref(0), - vable_w_globals: ctx.const_ref(0), + frame_w_globals: ctx.const_ref(0), }); + let ec_ref = ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); + sym.set_test_execution_context(ec_ref); let mut state = MIFrame::from_sym(&mut ctx, &mut sym, frame_ptr, resume_pc, resume_pc); state.set_resume_marker_for_test(resume_pc); @@ -15117,13 +15154,15 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Int], registers_r: vec![OpRef::NONE; max_color + 1], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(0), vable_pycode: ctx.const_ref(0), vable_valuestackdepth: ctx.const_int(0), vable_debugdata: ctx.const_ref(0), - vable_w_globals: ctx.const_ref(0), + frame_w_globals: ctx.const_ref(0), }); + let ec_ref = ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); + sym.set_test_execution_context(ec_ref); let mut state = MIFrame::from_sym(&mut ctx, &mut sym, frame_ptr, resume_pc, resume_pc); state.set_resume_marker_for_test(resume_pc); @@ -15234,13 +15273,16 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Ref, Type::Ref, Type::Ref], registers_r: vec![OpRef::NONE; 8], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(resume_pc as i64 - 1), vable_pycode: ctx.const_ref(frame.pycode as usize as i64), vable_valuestackdepth: ctx.const_int(7), vable_debugdata: ctx.const_ref(frame.debugdata as usize as i64), - vable_w_globals: ctx.const_ref(frame.w_globals as usize as i64), + frame_w_globals: ctx.const_ref(frame.get_w_globals() as usize as i64), }); + let ec_ref = + ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); + sym.set_test_execution_context(ec_ref); trace_state::seed_compiled_trace_jitcode_test_state( &mut sym, &mut ctx, @@ -15382,13 +15424,15 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Ref, Type::Ref, Type::Ref], registers_r: vec![OpRef::NONE; 8], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(resume_pc as i64 - 1), vable_pycode: ctx.const_ref(frame.pycode as usize as i64), vable_valuestackdepth: ctx.const_int(7), vable_debugdata: ctx.const_ref(frame.debugdata as usize as i64), - vable_w_globals: ctx.const_ref(frame.w_globals as usize as i64), + frame_w_globals: ctx.const_ref(frame.get_w_globals() as usize as i64), }); + let ec_ref = ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); + sym.set_test_execution_context(ec_ref); trace_state::seed_compiled_trace_jitcode_test_state( &mut sym, &mut ctx, @@ -15498,13 +15542,15 @@ mod tests { symbolic_stack_types: vec![Type::Ref, Type::Ref], registers_r: vec![local0, stack0, stack1], concrete_stack: vec![], - concrete_namespace: frame.w_globals, + concrete_namespace: frame.get_w_globals(), vable_last_instr: ctx.const_int(target_pc as i64 - 1), vable_pycode: ctx.const_ref(frame.pycode as usize as i64), vable_valuestackdepth: ctx.const_int(3), vable_debugdata: ctx.const_ref(frame.debugdata as usize as i64), - vable_w_globals: ctx.const_ref(frame.w_globals as usize as i64), + frame_w_globals: ctx.const_ref(frame.get_w_globals() as usize as i64), }); + let ec_ref = ctx.const_ref(pyre_interpreter::call::getexecutioncontext() as usize as i64); + sym.set_test_execution_context(ec_ref); let mut state = MIFrame::from_sym(&mut ctx, &mut sym, frame_ptr, target_pc, target_pc); let jump_args = state.capture_close_loop_args_at(Some(target_pc), None); diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index cdc99a22c4f..a2223f81c2c 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -1067,18 +1067,20 @@ fn derive_pc_live_indices_from_sparse( .collect() } -/// Derive the pre-merge `-live-` anchor that immediately precedes each -/// `catch_exception` in the canonical SSA representation. The anchor is keyed -/// by the Python PC whose `pc_first_insn_pos` range owns it, then +/// Derive every pre-merge trailing `-live-` anchor emitted after a call in the +/// canonical SSA representation. The anchor is keyed by the Python PC whose +/// `pc_first_insn_pos` range owns it, then /// `compute_liveness_with_pc_anchors` remaps it into the spliced instruction -/// stream used by the runtime's post-call catch marker. +/// stream used by the runtime's post-call resume marker. /// /// `derive_after_call_indices_from_sparse` stores one anchor per Python PC. -/// Multiple `catch_exception` sites owned by one PC would overwrite that entry. -/// The representation is sound because `catch_exception` is emitted -/// once for each can-raise block exit, while additional catch links from a -/// multi-exit block lower through `make_exception_link`, which emits no -/// `catch_exception`. +/// The representation is sound because one lowered Python opcode emits at most +/// one call requiring a trailing marker. Do not infer these anchors from +/// `catch_exception`: `jtransform.py::handle_residual_call` also appends the +/// marker for a can-raise call with no local exception handler. Reuse the same +/// `insn_needs_trailing_live` predicate that emits the marker so the producer +/// and consumer cannot disagree about which call sites need a post-call resume +/// coordinate. fn derive_after_call_indices_from_sparse( ssarepr: &super::flatten::SSARepr, n_pcs: usize, @@ -1086,19 +1088,19 @@ fn derive_after_call_indices_from_sparse( let mut out: Vec> = vec![None; n_pcs]; let pc_pos = sparse_pc_owner_table(ssarepr); for (q, insn) in ssarepr.insns.iter().enumerate() { - let is_catch = matches!( - insn, - super::flatten::Insn::Op { opname, .. } if opname == "catch_exception" - ); - if !is_catch { + if !insn.is_live() { continue; } - let Some(live_pos) = q.checked_sub(1).filter(|&i| ssarepr.insns[i].is_live()) else { + let Some(call_pos) = q + .checked_sub(1) + .filter(|&i| super::flatten::insn_needs_trailing_live(&ssarepr.insns[i])) + else { continue; }; - if let Some(pc) = sparse_owner_pc(&pc_pos, live_pos) { + debug_assert_eq!(call_pos + 1, q); + if let Some(pc) = sparse_owner_pc(&pc_pos, q) { if pc < n_pcs { - out[pc] = Some(live_pos); + out[pc] = Some(q); } } } @@ -5323,7 +5325,6 @@ fn filter_liveness_in_place( if any_reachable && let Some(extra) = catch_extra_refs.get(&insn_idx) { union_r.extend(extra.iter().copied()); } - // #348 Part (2): the marker's Ref colors are now final in `union_r`. // Collect the group's `(color, slot)` entries (union of member PCs' // per-PC maps) restricted to those colors, then publish to every @@ -6049,13 +6050,12 @@ impl CodeWriter { // through `VABLEINFO.static_field_descrs` since each backend may // reorder fields. Pyre's `_virtualizable_` order matches PyPy // `interp_jit.py:25-30` line by line: - // [last_instr, pycode, valuestackdepth, debugdata, w_globals], + // [last_instr, pycode, valuestackdepth, debugdata], // so the literals match // `virtualizable_spec.rs::PYFRAME_VABLE_FIELDS`. const VABLE_LAST_INSTR_FIELD_IDX: u16 = 0; const VABLE_CODE_FIELD_IDX: u16 = 1; const VABLE_VALUESTACKDEPTH_FIELD_IDX: u16 = 2; - const VABLE_NAMESPACE_FIELD_IDX: u16 = 4; // regalloc.py: compile-time stack depth counter — tracks which // stack register (stack_base + depth) is the current TOS. @@ -9504,17 +9504,16 @@ impl CodeWriter { // const-folded loads into five residual calls and // takes `guard_failures` from 1803 to 63769. let result_value: super::flow::FlowValue = if is_true_portal { - let ns_var = emit_graph_op_with_result( - &mut graph, - ¤t_block.block(), - "getfield_vable_r", - vable_getfield_ref_graph_args( - frame_var.into(), - VABLE_NAMESPACE_FIELD_IDX, - ), - Kind::Ref, - py_pc as i64, - ); + // `bh_load_global_fn` derives globals from the + // live frame; the namespace operand is only a + // trace-time cell-fold hint. The walker replaces + // this null placeholder with this MIFrame's + // `get_w_globals()` result before recording. + let ns_var: super::flow::FlowValue = super::flow::Constant::new( + super::flow::ConstantValue::Signed(0), + Some(Kind::Ref), + ) + .into(); let code_const: super::flow::FlowValue = super::flow::Constant::new( super::flow::ConstantValue::Signed(w_code as i64), @@ -16165,8 +16164,8 @@ pub fn register_portal_jitdriver(code: &pyre_interpreter::CodeObject) -> bool { /// `PyFrame::call`). /// * `bh_normalize_raise_varargs_with_frame(frame_ptr, exc, cause)` — /// `frame_ptr` non-null asserted; pins -/// `(*parent_frame_ptr).execution_context` for the normalization -/// path. Same migration shape as `bh_call_fn_*`. Not yet migrated. +/// the current activation's execution context for normalization. +/// Same migration shape as `bh_call_fn_*`. Not yet migrated. /// /// Today the un-migrated emit sites are latent for non-portal callees: /// production tracing records IR ops symbolically. The full-body-walk @@ -17378,6 +17377,33 @@ mod tests { assert_eq!(state.variable_slot(&v_absent), None); } + /// `jtransform.py::handle_residual_call` emits a trailing `-live-` for a + /// can-raise residual call even when the graph has no local exception + /// handler. The after-call anchor census must therefore follow the call + /// predicate, not look for a following `catch_exception`. + #[test] + fn after_call_anchor_includes_canraise_call_without_catch_exception() { + let mut ssarepr = SSARepr::new("canraise_without_catch"); + ssarepr.pc_first_insn_pos.push((0, 0)); + ssarepr + .insns + .push(super::super::flatten::build_binary_op_residual_call_ir_r_insn(11, 0, 0, 1, 2)); + ssarepr.insns.push(Insn::live(Vec::new())); + + assert!( + !ssarepr.insns.iter().any(|insn| matches!( + insn, + Insn::Op { opname, .. } if opname == "catch_exception" + )), + "fixture must exercise a call with no local exception handler", + ); + assert_eq!( + derive_after_call_indices_from_sparse(&ssarepr, 1), + vec![Some(1)], + "the structural trailing-live marker must be the PC's after-call anchor", + ); + } + /// `liveness.py:5-12` expands a `-live-` marker to "all values that are /// alive at this point", under no frame-slot condition: a Ref color that /// is SSA-live across the marker but names no frame slot at that PC stays diff --git a/pyre/pyre-jit/src/jit/cpu.rs b/pyre/pyre-jit/src/jit/cpu.rs index c00a4d4e503..1327598f938 100644 --- a/pyre/pyre-jit/src/jit/cpu.rs +++ b/pyre/pyre-jit/src/jit/cpu.rs @@ -374,8 +374,8 @@ pub struct Cpu { /// `bhimpl_build_slice` — (argc, start, stop, step) → new slice. pub build_slice_fn: extern "C" fn(i64, i64, i64, i64) -> i64, /// `RAISE_VARARGS` normalization helper used before `raise/r`. - /// `(frame: Ref, exc: Ref, cause: Ref) → Ref` — the explicit frame - /// pointer feeds `frame.execution_context` directly. + /// `(frame: Ref, exc: Ref, cause: Ref) → Ref` — normalization reads the + /// current activation's execution context, as `space.getexecutioncontext()`. pub normalize_raise_varargs_fn: extern "C" fn(i64, i64, i64) -> i64, /// Read per-thread `CURRENT_EXCEPTION` — used by `PUSH_EXC_INFO`. pub get_current_exception_fn: extern "C" fn() -> i64, diff --git a/pyre/pyre-jit/src/jit/flatten.rs b/pyre/pyre-jit/src/jit/flatten.rs index de13fe952f3..bdc5773fee6 100644 --- a/pyre/pyre-jit/src/jit/flatten.rs +++ b/pyre/pyre-jit/src/jit/flatten.rs @@ -2577,7 +2577,7 @@ fn regalloc_color( /// is exactly `calldescr_canraise` = `effect_info.check_can_raise(false)` /// (`call.py calldescr_canraise` -> `effectinfo.py:232 /// check_can_raise`), read off the trailing `CallDescrStub` operand. -fn insn_needs_trailing_live(insn: &Insn) -> bool { +pub(super) fn insn_needs_trailing_live(insn: &Insn) -> bool { let Insn::Op { opname, args, .. } = insn else { return false; }; diff --git a/pyre/pyre-object/Cargo.toml b/pyre/pyre-object/Cargo.toml index 36ae21412a5..5d5e3148b02 100644 --- a/pyre/pyre-object/Cargo.toml +++ b/pyre/pyre-object/Cargo.toml @@ -9,6 +9,7 @@ description = "Python object model for pyre interpreter" [dependencies] num-traits = { workspace = true } majit-gc = { workspace = true } +majit-ir = { workspace = true } majit-rlib = { workspace = true } parking_lot = { workspace = true } majit-macros = { workspace = true } diff --git a/pyre/pyre-object/src/celldict.rs b/pyre/pyre-object/src/celldict.rs index 0a47ae703a5..47ac2ec004b 100644 --- a/pyre/pyre-object/src/celldict.rs +++ b/pyre/pyre-object/src/celldict.rs @@ -1377,7 +1377,8 @@ mod tests { use std::sync::atomic::{AtomicBool, Ordering}; let mut strategy = ModuleDictStrategy::new(); let flag = Arc::new(AtomicBool::new(false)); - strategy.current_version_qmut().register_loop_token(&flag); + let token = crate::quasiimmut::test_loop_token(&flag); + strategy.current_version_qmut().register_loop_token(&token); // Before a structural change the watching loop is still valid. assert!(!flag.load(Ordering::Acquire)); // `mutated()` reassigns `version` and must invalidate the loop. @@ -1391,9 +1392,10 @@ mod tests { use std::sync::atomic::AtomicBool; let mut strategy = ModuleDictStrategy::new(); let flag = Arc::new(AtomicBool::new(false)); - strategy.current_version_qmut().register_loop_token(&flag); + let token = crate::quasiimmut::test_loop_token(&flag); + strategy.current_version_qmut().register_loop_token(&token); // Drop the only strong ref: the weak watcher can no longer upgrade. - drop(flag); + drop(token); // notify (via mutated) must not panic, and `_invalidate_now` unlinks // the instance whether or not any flag could still be upgraded. strategy.mutated(); diff --git a/pyre/pyre-object/src/quasiimmut.rs b/pyre/pyre-object/src/quasiimmut.rs index c494ad05d2f..7870f31d158 100644 --- a/pyre/pyre-object/src/quasiimmut.rs +++ b/pyre/pyre-object/src/quasiimmut.rs @@ -16,9 +16,10 @@ use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; /// `quasiimmut.py QuasiImmut` — the loops that baked one quasi-immutable /// field's value as a constant, and must be revoked when it changes. /// -/// The flag stands in for upstream's `looptoken` + `cpu.invalidate_loop`: the -/// backend already routes `GUARD_NOT_INVALIDATED` through a per-artifact -/// `AtomicBool`, so setting it is what `looptoken.invalidated = True` buys. +/// Like upstream, the registry holds the owning loop token, not a particular +/// loop/bridge machine-code fragment. Invalidating the token both removes it +/// from warm entry and activates every still-unpatched +/// `GUARD_NOT_INVALIDATED` generation owned by that token. /// /// Upstream reaches an instance twice — `QuasiImmutDescr.__init__` binds it /// while recording (`pyjitpl.py:1081`) and `compile.py:204-207` registers the @@ -41,7 +42,7 @@ pub struct QuasiImmut { struct LoopTokens { /// `quasiimmut.py:61-63` — weak so a retired loop drops out instead of /// being kept alive by the object whose field it read. - looptokens_wrefs: Vec>, + looptokens_wrefs: Vec>, /// `quasiimmut.py compress_limit = 30`. compress_limit: usize, } @@ -53,8 +54,8 @@ impl LoopTokens { /// unbounded list. /// /// Upstream's note that already-invalidated tokens must be kept applies - /// here too: the flag stays live while its artifact does, and re-flipping - /// an already-set flag is what keeps a multiply-invalidated loop revoked. + /// here too: a repeated invalidation must reach the same token again so + /// the backend can activate guards compiled since the previous call. fn compress(&mut self) { self.looptokens_wrefs.retain(|w| w.strong_count() > 0); self.compress_limit = (self.looptokens_wrefs.len() + 15) * 2; @@ -95,16 +96,16 @@ impl QuasiImmut { /// its `GUARD_NOT_INVALIDATED` has to fail on the first entry, since the /// value it baked is the pre-mutation one and no later sweep will reach a /// list this instance has already emptied. - pub fn register_loop_token(&self, flag: &Arc) { + pub fn register_loop_token(&self, token: &Arc) { let mut tokens = self.tokens.lock(); if self.unlinked.load(Ordering::Acquire) { - flag.store(true, Ordering::Release); + token.invalidate_for_quasi_immut(); return; } if tokens.looptokens_wrefs.len() > tokens.compress_limit { tokens.compress(); } - tokens.looptokens_wrefs.push(Arc::downgrade(flag)); + tokens.looptokens_wrefs.push(Arc::downgrade(token)); } /// `quasiimmut.py invalidate` — every loop recorded here becomes @@ -128,8 +129,8 @@ impl QuasiImmut { std::mem::take(&mut tokens.looptokens_wrefs) }; for wref in wrefs { - if let Some(flag) = wref.upgrade() { - flag.store(true, Ordering::Release); + if let Some(token) = wref.upgrade() { + token.invalidate_for_quasi_immut(); } } } @@ -289,6 +290,28 @@ pub unsafe fn sweep_quasi_immut_field(field: *const QuasiImmutField) { unsafe { (*field).invalidate() }; } +#[cfg(test)] +#[derive(Debug)] +pub(crate) struct TestLoopToken { + invalidated: Arc, +} + +#[cfg(test)] +impl majit_ir::QuasiImmutLoopToken for TestLoopToken { + fn invalidate_for_quasi_immut(&self) { + self.invalidated.store(true, Ordering::Release); + } +} + +#[cfg(test)] +pub(crate) fn test_loop_token( + invalidated: &Arc, +) -> Arc { + Arc::new(TestLoopToken { + invalidated: invalidated.clone(), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -300,9 +323,11 @@ mod tests { let qi = QuasiImmut::new(); let live = Arc::new(AtomicBool::new(false)); let dead = Arc::new(AtomicBool::new(false)); - qi.register_loop_token(&live); - qi.register_loop_token(&dead); - drop(dead); + let live_token = test_loop_token(&live); + let dead_token = test_loop_token(&dead); + qi.register_loop_token(&live_token); + qi.register_loop_token(&dead_token); + drop(dead_token); qi.invalidate(); assert!(live.load(Ordering::Acquire), "a live loop must be revoked"); @@ -328,7 +353,8 @@ mod tests { // Each "recompile" drops its artifact immediately, so every // registered weak ref is already dead by the next round. let flag = Arc::new(AtomicBool::new(false)); - qi.register_loop_token(&flag); + let token = test_loop_token(&flag); + qi.register_loop_token(&token); } assert!( qi.len() <= qi.compress_limit() + 1, @@ -349,7 +375,10 @@ mod tests { assert!(!field.is_installed()); let flag = Arc::new(AtomicBool::new(false)); - field.get_current_qmut_instance().register_loop_token(&flag); + let token = test_loop_token(&flag); + field + .get_current_qmut_instance() + .register_loop_token(&token); assert!(field.is_installed()); field.invalidate(); @@ -363,9 +392,10 @@ mod tests { field.invalidate(); let flag2 = Arc::new(AtomicBool::new(false)); + let token2 = test_loop_token(&flag2); field .get_current_qmut_instance() - .register_loop_token(&flag2); + .register_loop_token(&token2); assert!(field.is_installed()); field.invalidate(); assert!(flag2.load(Ordering::Acquire)); @@ -393,7 +423,8 @@ mod tests { // Registering on the swept instance cannot be silently lost: the loop // folded a pre-mutation value, so it is born invalid. let flag = Arc::new(AtomicBool::new(false)); - recorded.register_loop_token(&flag); + let token = test_loop_token(&flag); + recorded.register_loop_token(&token); assert!(flag.load(Ordering::Acquire)); assert_eq!(recorded.len(), 0, "a swept list must not grow again"); } @@ -427,7 +458,8 @@ mod tests { // and the registration is recorded rather than born-invalid. assert!(recorded.is_current()); let flag = Arc::new(AtomicBool::new(false)); - recorded.register_loop_token(&flag); + let token = test_loop_token(&flag); + recorded.register_loop_token(&token); assert!(!flag.load(Ordering::Acquire)); assert_eq!(recorded.len(), 1); } @@ -454,8 +486,11 @@ mod tests { let stop = &stop; scope.spawn(move || { let flag = Arc::new(AtomicBool::new(false)); + let token = test_loop_token(&flag); while !stop.load(Ordering::Relaxed) { - field.get_current_qmut_instance().register_loop_token(&flag); + field + .get_current_qmut_instance() + .register_loop_token(&token); } }); } diff --git a/pyre/pyre-object/src/typeobject.rs b/pyre/pyre-object/src/typeobject.rs index bc83ee8da33..416e49b50f2 100644 --- a/pyre/pyre-object/src/typeobject.rs +++ b/pyre/pyre-object/src/typeobject.rs @@ -2189,9 +2189,10 @@ mod tests { ); let flag = Arc::new(AtomicBool::new(false)); + let token = crate::quasiimmut::test_loop_token(&flag); unsafe { w_type_current_qmut_instance(obj) } .expect("a type resolves an instance") - .register_loop_token(&flag); + .register_loop_token(&token); assert!(w_type.quasi_immut_watchers.is_installed()); unsafe { w_type_set_version_tag(obj, new_version_tag()) }; diff --git a/pyre/pyre-wasm-test/src/main.rs b/pyre/pyre-wasm-test/src/main.rs index e9cebc37bf8..ae988fccf23 100644 --- a/pyre/pyre-wasm-test/src/main.rs +++ b/pyre/pyre-wasm-test/src/main.rs @@ -23,6 +23,11 @@ fn run_test(name: &str, source: &str, expected: &str) { }; let execution_context = std::rc::Rc::new(PyExecutionContext::default()); + // `threadlocals.py enter_thread` — the ExecutionContext slot belongs to the + // OS-thread locals, and each launcher installs it before running anything. + // Left null, `eval_frame_plain_with_resume` takes its context-free arm: no + // `enter`/`leave`, no `call_trace`, so `sys.settrace` is silently inert. + pyre_interpreter::call::set_last_exec_ctx(std::rc::Rc::as_ptr(&execution_context)); let mut frame = match pyframe::PyFrame::new_with_context(code, execution_context) { Ok(frame) => frame, Err(e) => {