jit: inline a user __next__ under FOR_ITER, plus three exception wrong-code fixes

.github/workflows/pyre-ci.yml

@@ -726,8 +726,12 @@ jobs:
       # the wasm32 target above; WASM_MODULE_PATH is the .wasm-host.wasm the test
       # loads), so run the ignored tests against them here. The plain
       # `cargo test --all` job cannot: it never builds those artifacts.
+      #
+      # Serialized: each test spawns a full pyre process on both backends, and
+      # the default one-thread-per-core fan-out ran them concurrently. Two runs
+      # then lost a runner with an empty stderr, in a different test each time.
       if: runner.os == 'Linux'
-      run: cargo test -p majit-backend-wasm --test codegen_test -- --ignored
+      run: cargo test -p majit-backend-wasm --test codegen_test -- --ignored --test-threads=1
     - name: Run release-only dispatcher-graph acceptance test (Linux only)
       # `slow_generated_jitcodes_preserve_complete_dispatcher_graph` self-ignores
       # under `debug_assertions`, so the `cargo test --all` job never runs it: it

majit/majit-backend/src/resume_guard_descr.rs

@@ -264,6 +264,11 @@ pub struct ResumeGuardDescr {
     /// longer depends on the guard's per-trace fail index.  `0` = not a
     /// range guard.
     pub range_foriter_key: AtomicU64,
+    /// Pyre-only: FOR_ITER green key for guards emitted while inlining a user
+    /// instance's `__next__`.  A bridge from one of these guards must retain
+    /// the generic `jit_next` conversion path when it re-enters FOR_ITER.
+    /// `0` means this descr did not originate in that inline route.
+    pub instance_next_foriter_key: AtomicU64,
 }
 
 // Safety: single-threaded JIT (RPython GIL parity).
@@ -289,6 +294,12 @@ impl Descr for ResumeGuardDescr {
             key => Some(key),
         }
     }
+    fn instance_next_foriter_green_key(&self) -> Option<u64> {
+        match self.instance_next_foriter_key.load(Ordering::Relaxed) {
+            0 => None,
+            key => Some(key),
+        }
+    }
     /// compile.py:844-846: ResumeGuardDescr.clone()
     fn clone_descr(&self) -> Option<DescrRef> {
         Some(Arc::new(ResumeGuardDescr {
@@ -316,9 +327,12 @@ impl Descr for ResumeGuardDescr {
             bridge_body_ptr_cache: Box::new(AtomicUsize::new(0)),
             bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
             bridge_dispatch_drop_fn: OnceLock::new(),
-            // The clone guards the same range site; preserve the tag so a
-            // cloned range class guard still demotes on failure.
+            // The clone guards the same FOR_ITER site; preserve either tag so
+            // guard-failure routing survives guard copying.
             range_foriter_key: AtomicU64::new(self.range_foriter_key.load(Ordering::Relaxed)),
+            instance_next_foriter_key: AtomicU64::new(
+                self.instance_next_foriter_key.load(Ordering::Relaxed),
+            ),
         }))
     }
 }
@@ -600,6 +614,7 @@ pub fn make_resume_guard_descr_typed(types: Vec<Type>) -> DescrRef {
         bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
         bridge_dispatch_drop_fn: OnceLock::new(),
         range_foriter_key: AtomicU64::new(0),
+        instance_next_foriter_key: AtomicU64::new(0),
     })
 }
 

majit/majit-gc/src/collector.rs

@@ -3717,10 +3717,20 @@ impl MiniMarkGC {
             None => {
                 let type_info = self.types.get(type_id);
                 let length = unsafe { *((obj_addr + type_info.length_offset) as *const usize) };
+                // `set_forwarding_address` stores the new address in the word
+                // right after the header — `obj_addr + 0`.  Every type whose
+                // `length_offset` is 0 therefore has its length word overwritten
+                // the moment it is forwarded, and `ItemsBlock` is one
+                // (`ITEMS_BLOCK_LEN_OFFSET` is `capacity`, its first field).  A
+                // length that is a plausible heap address is that corpse, read
+                // by a path that skipped the `is_forwarded` check, not an
+                // uninitialized allocation — name which one this is rather than
+                // leaving both readings open.
+                let forwarded = unsafe { (*header_of(obj_addr)).is_forwarded() };
                 panic!(
                     "GC BUG: varsize length describes no allocation: length={} (read at \
                      obj_addr={:#x} + length_offset={}) item_size={} fixed_size={} \
-                     type_id={} header_addr={:#x} nursery_start={:#x} site={}",
+                     type_id={} header_addr={:#x} nursery_start={:#x} forwarded={} site={}",
                     length,
                     obj_addr,
                     type_info.length_offset,
@@ -3729,6 +3739,7 @@ impl MiniMarkGC {
                     type_id,
                     obj_addr - GcHeader::SIZE,
                     self.nursery.start_ptr() as usize,
+                    forwarded,
                     site,
                 );
             }

majit/majit-gc/src/header.rs

@@ -96,6 +96,12 @@ impl GcHeader {
     /// `hdr + SIZE` lies outside the single-field extent a `&mut GcHeader`
     /// reference is allowed to touch under Rust's aliasing model.
     ///
+    /// `hdr + SIZE` is the object's own first payload word, so a varsize type
+    /// registered with `length_offset == 0` — `ItemsBlock`, whose `capacity`
+    /// is its first field — has its length destroyed here.  Any size read of a
+    /// forwarded object of such a type returns the forwarding address in place
+    /// of the length; check `is_forwarded` and follow it first.
+    ///
     /// # Safety
     /// `hdr` must point to a valid `GcHeader` followed by at least
     /// `size_of::<usize>()` bytes of writable memory, and no other reference

majit/majit-ir/src/descr.rs

@@ -3599,6 +3599,14 @@ pub trait Descr: Send + Sync + std::fmt::Debug {
             .and_then(|prev| prev.range_foriter_green_key())
     }
 
+    /// Pyre-only: the FOR_ITER green key whose user-instance `__next__`
+    /// inline emitted this guard, or `None`.  Copied guards chase their donor
+    /// descr exactly like [`Descr::range_foriter_green_key`].
+    fn instance_next_foriter_green_key(&self) -> Option<u64> {
+        self.prev_descr()
+            .and_then(|prev| prev.instance_next_foriter_green_key())
+    }
+
     /// intbounds.py: descr.is_integer_bounded() / get_integer_min/max.
     /// Returns (field_size_bytes, is_signed) if this is a field descriptor.
     /// Used by intbounds to narrow GETFIELD result bounds.

majit/majit-ir/src/effectinfo.rs

@@ -935,7 +935,10 @@ pub enum PyreHelperKind {
     /// raise instead of declining to the trait.
     RaiseVarargs,
     /// `get_current_exception()` — the PUSH_EXC_INFO `prev = ec.sys_exc_value`
-    /// save residual (`() → Ref`, TLS read via `cpu.get_current_exception_fn`).
+    /// save residual, and the read a catch-covered bare `raise` uses to obtain
+    /// the exception it re-raises (`() → Ref`, TLS read via
+    /// `cpu.get_current_exception_fn`).  Only the save is followed by a store
+    /// and a matching POP_EXCEPT restore.
     /// The full-body walker recognises this tag to
     /// lower it to `GETFIELD_GC_R(ec, sys_exc_value)` so the exc-info save
     /// participates in the balanced save/restore the heap optimizer
@@ -958,6 +961,9 @@ pub enum PyreHelperKind {
     /// the in-flight iteration to the live frame instead of dropping it (the
     /// iterator advance is an irreversible side effect with no journal undo).
     ForIterNext,
+    /// `jit_exception_match(exc, match_class)` — the infallible Python-level
+    /// exception MRO test used by FOR_ITER's materialized catch arm.
+    ForIterExceptionMatch,
     /// `get_iter(obj)` — the GET_ITER residual (`iter(obj)`).  The full-body
     /// walker recognises exact machine-word `range` objects and emits the
     /// virtual `W_IntRangeIterator` allocation shape directly.
@@ -1054,9 +1060,11 @@ pub enum PyreHelperKind {
     MakeFunction,
     /// `bh_clear_in_flight_exception()` — the `[] -> void` residual emitted by
     /// PUSH_EXC_INFO to complete the caught-exception ownership transfer.  The
-    /// full-body walker recognises this tag to keep the executed-effect
-    /// odometer off it: the written slot is a GC-liveness root with no value
-    /// reader, so a non-committing walk has nothing to undo.
+    /// full-body walker applies the concrete clear during its authoritative
+    /// walk and emits no runtime IR: compiled traceback construction never
+    /// publishes this interpreter-only GC-liveness carrier.  Generic fallback
+    /// also keeps the executed-effect odometer off it because the slot has no
+    /// value reader and a non-committing walk has nothing to undo.
     ClearInFlightException,
 }
 

majit/majit-metainterp/src/compile.rs

@@ -3242,6 +3242,7 @@ pub fn make_fail_descr_with_index(fail_index: u32, num_live: usize) -> DescrRef
         bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
         bridge_dispatch_drop_fn: OnceLock::new(),
         range_foriter_key: AtomicU64::new(0),
+        instance_next_foriter_key: AtomicU64::new(0),
     })
 }
 
@@ -3335,6 +3336,7 @@ pub fn make_resume_guard_descr_typed(types: Vec<Type>) -> DescrRef {
         bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
         bridge_dispatch_drop_fn: OnceLock::new(),
         range_foriter_key: AtomicU64::new(0),
+        instance_next_foriter_key: AtomicU64::new(0),
     })
 }
 
@@ -3357,6 +3359,52 @@ pub fn make_resume_guard_descr_range_foriter(green_key: u64) -> DescrRef {
     descr
 }
 
+/// Tag a guard emitted while inlining a user instance's `__next__` with the
+/// caller FOR_ITER key.  A guard-failure bridge uses the tag to keep the
+/// exception-to-exhaustion conversion on the generic residual path.
+///
+/// `opcode` selects the subtype `compile.py:924-942 invent_fail_descr_for_op`
+/// would have minted.  Stamping this descr fills `op.getdescr()`, and
+/// `store_final_boxes_in_guard` only invents on the empty arm — so a marker
+/// minted as a plain `ResumeGuardDescr` would cost a `GUARD_NOT_FORCED` its
+/// `is_guard_forced()` (which vetoes bridge compilation) or a
+/// `GUARD_NO_EXCEPTION` its `is_guard_exc()` (which routes the pending
+/// exception).  The whole inlined `__next__` body is tagged, residual guards
+/// included, so both opcodes reach here.
+pub fn make_resume_guard_descr_instance_next_foriter(
+    opcode: Option<OpCode>,
+    green_key: u64,
+) -> DescrRef {
+    let descr = match opcode {
+        Some(OpCode::GuardNotForced | OpCode::GuardNotForced2) => {
+            make_resume_guard_forced_descr_typed(Vec::new())
+        }
+        Some(OpCode::GuardException | OpCode::GuardNoException) => {
+            make_resume_guard_exc_descr_typed(Vec::new())
+        }
+        _ => make_resume_guard_descr_typed(Vec::new()),
+    };
+    resume_guard_inner(&descr)
+        .expect("every arm above constructs a ResumeGuardDescr or a newtype over one")
+        .instance_next_foriter_key
+        .store(green_key, Ordering::Relaxed);
+    descr
+}
+
+/// The `ResumeGuardDescr` inside a descr that either is one or is one of its
+/// tag-only newtypes.
+fn resume_guard_inner(descr: &DescrRef) -> Option<&ResumeGuardDescr> {
+    let any = descr.as_any()?;
+    if let Some(plain) = any.downcast_ref::<ResumeGuardDescr>() {
+        return Some(plain);
+    }
+    if let Some(forced) = any.downcast_ref::<ResumeGuardForcedDescr>() {
+        return Some(&forced.inner);
+    }
+    any.downcast_ref::<ResumeGuardExcDescr>()
+        .map(|exc| &exc.inner)
+}
+
 /// compile.py:892: ResumeAtPositionDescr(ResumeGuardDescr) — subclass
 /// with no additional fields or method overrides. Type tag only.
 ///
@@ -3616,6 +3664,7 @@ pub fn make_resume_at_position_descr_typed(types: Vec<Type>) -> DescrRef {
             bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
             bridge_dispatch_drop_fn: OnceLock::new(),
             range_foriter_key: AtomicU64::new(0),
+            instance_next_foriter_key: AtomicU64::new(0),
         },
     })
 }
@@ -3665,6 +3714,18 @@ impl majit_ir::Descr for ResumeGuardForcedDescr {
     fn is_resume_guard(&self) -> bool {
         true
     }
+    /// Subclassing in RPython keeps the base attributes readable; a Rust
+    /// newtype only exposes what it forwards, and the default accessor walks
+    /// `prev_descr`, which a wrapper does not have.  Forward both walker
+    /// marker keys explicitly: the FOR_ITER routes key failure handling on
+    /// them, and `store_final_boxes_in_guard` re-mints a marked descr on
+    /// unroll's second emission only while it can still read the key.
+    fn range_foriter_green_key(&self) -> Option<u64> {
+        self.inner.range_foriter_green_key()
+    }
+    fn instance_next_foriter_green_key(&self) -> Option<u64> {
+        self.inner.instance_next_foriter_green_key()
+    }
     /// compile.py:873-876 ResumeGuardDescr.clone() — `ResumeGuardForcedDescr`
     /// inherits the base implementation (no override at compile.py:939+),
     /// so cloning produces a plain `ResumeGuardDescr` with resume attributes
@@ -3884,6 +3945,7 @@ pub fn make_resume_guard_forced_descr_typed(types: Vec<Type>) -> DescrRef {
             bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
             bridge_dispatch_drop_fn: OnceLock::new(),
             range_foriter_key: AtomicU64::new(0),
+            instance_next_foriter_key: AtomicU64::new(0),
         },
     })
 }
@@ -3917,6 +3979,15 @@ impl majit_ir::Descr for ResumeGuardExcDescr {
     fn is_resume_guard(&self) -> bool {
         true
     }
+    /// The `ResumeGuardForcedDescr` reasoning applies here unchanged: a
+    /// newtype exposes only what it forwards, and both walker marker keys
+    /// have to stay readable through it.
+    fn range_foriter_green_key(&self) -> Option<u64> {
+        self.inner.range_foriter_green_key()
+    }
+    fn instance_next_foriter_green_key(&self) -> Option<u64> {
+        self.inner.instance_next_foriter_green_key()
+    }
     /// compile.py:881-882 `class ResumeGuardExcDescr(ResumeGuardDescr): pass`
     /// — no clone() override, so inheriting compile.py:873-876
     /// `ResumeGuardDescr.clone()` produces a plain `ResumeGuardDescr` with
@@ -4136,6 +4207,7 @@ pub fn make_resume_guard_exc_descr_typed(types: Vec<Type>) -> DescrRef {
             bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
             bridge_dispatch_drop_fn: OnceLock::new(),
             range_foriter_key: AtomicU64::new(0),
+            instance_next_foriter_key: AtomicU64::new(0),
         },
     })
 }
@@ -5132,6 +5204,7 @@ impl majit_ir::Descr for CompileLoopVersionDescr {
                 bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
                 bridge_dispatch_drop_fn: OnceLock::new(),
                 range_foriter_key: AtomicU64::new(0),
+                instance_next_foriter_key: AtomicU64::new(0),
             },
         }))
     }
@@ -5353,6 +5426,7 @@ fn make_compile_loop_version_descr_with_payload(types: Vec<Type>, payload: RdPay
             bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
             bridge_dispatch_drop_fn: OnceLock::new(),
             range_foriter_key: AtomicU64::new(0),
+            instance_next_foriter_key: AtomicU64::new(0),
         },
     })
 }
@@ -5874,6 +5948,7 @@ mod fail_descr_tests {
                 bridge_dispatch_cell: AtomicPtr::new(std::ptr::null_mut()),
                 bridge_dispatch_drop_fn: OnceLock::new(),
                 range_foriter_key: AtomicU64::new(0),
+                instance_next_foriter_key: AtomicU64::new(0),
             },
         }) as DescrRef;
         let lv_fi = lv.index();
@@ -5972,6 +6047,34 @@ mod fail_descr_tests {
     }
 
     /// compile.py:832-851 ResumeGuardCopiedDescr(prev) parity:
+    /// The instance-next FOR_ITER marker key must stay readable no matter
+    /// which subtype the guard's opcode selects.  Two consumers depend on it:
+    /// guard-failure routing keys the FOR_ITER handling on it, and
+    /// `store_final_boxes_in_guard` re-mints a marked descr for unroll's
+    /// second emission only while it can still read the key — without that
+    /// re-mint the second emission finalizes an already-finalized descr and
+    /// trips the once-per-descr `finish()` assert.
+    #[test]
+    fn test_instance_next_marker_survives_every_guard_subtype() {
+        const KEY: u64 = 0xF0_1D_ED;
+        for opcode in [
+            None,
+            Some(OpCode::GuardNotForced),
+            Some(OpCode::GuardNotForced2),
+            Some(OpCode::GuardException),
+            Some(OpCode::GuardNoException),
+            Some(OpCode::GuardClass),
+        ] {
+            let descr = make_resume_guard_descr_instance_next_foriter(opcode, KEY);
+            assert_eq!(
+                descr.instance_next_foriter_green_key(),
+                Some(KEY),
+                "opcode {opcode:?} minted a descr whose marker key is unreadable"
+            );
+            assert!(descr.is_resume_guard(), "opcode {opcode:?}");
+        }
+    }
+
     /// `get_resumestorage()` chases to `prev`, `fail_arg_types`
     /// shares the donor's vector, `is_resume_guard_copied()` flags
     /// the subtype, and the exc variant additionally reports

majit/majit-metainterp/src/lib.rs

@@ -149,7 +149,7 @@ pub use call_descr::{
 };
 pub use compile::{
     make_fail_descr, make_fail_descr_typed, make_finish_fail_descr_typed,
-    make_resume_guard_descr_range_foriter,
+    make_resume_guard_descr_instance_next_foriter, make_resume_guard_descr_range_foriter,
 };
 pub use io_buffer::{
     emit_commit_io, encode_decimal_i64, io_buffer_commit, io_buffer_discard, io_buffer_write,

majit/majit-metainterp/src/optimizeopt/mod.rs

@@ -6300,20 +6300,28 @@ impl OptContext {
             op.getdescr().is_some_and(|d| d.is_resume_guard_copied())
         );
 
-        // A walker-native range FOR_ITER class guard carries a pre-minted
-        // marker descr (`range_foriter_green_key`) so its failure can demote
-        // the specialization by descr identity.  `Op::clone` shares the descr
-        // Arc, so unroll's phase-1/phase-2 emissions of the guard reach this
-        // function on the same, already-finalized descr.  Mint a fresh marked
-        // descr for this emission — mirroring the `op.descr.is_none()` arm's
-        // fresh-per-emission descr — so the once-per-descr `finish()`
-        // invariant below still holds for it (and for every other guard).
+        // Walker-native FOR_ITER guards can carry a pre-minted range or
+        // user-instance-next marker descr so failure routing is keyed by descr
+        // identity.  `Op::clone` shares the descr Arc, so unroll's
+        // phase-1/phase-2 emissions of the guard reach this function on the
+        // same, already-finalized descr.  Mint a fresh marked descr for this
+        // emission — mirroring the `op.descr.is_none()` arm's fresh-per-emission
+        // descr — so the once-per-descr `finish()` invariant below still holds
+        // for it (and for every other guard).
         let refinalize_marked_key = op
             .getdescr()
             .and_then(|d| d.range_foriter_green_key())
             .filter(|_| op.resolved_rd_numb().is_some());
+        let refinalize_instance_next_key = op
+            .getdescr()
+            .and_then(|d| d.instance_next_foriter_green_key())
+            .filter(|_| op.resolved_rd_numb().is_some());
         if let Some(key) = refinalize_marked_key {
             op.setdescr(crate::compile::make_resume_guard_descr_range_foriter(key));
+        } else if let Some(key) = refinalize_instance_next_key {
+            op.setdescr(
+                crate::compile::make_resume_guard_descr_instance_next_foriter(Some(op.opcode), key),
+            );
         }
 
         // resume.py:397 `assert not storage.rd_numb` — finish() runs at