majit: evict per-loop side tables on retirement and harden MAJIT_BRIDGE_ONLY parsing

majit/majit-backend-cranelift/src/compiler.rs

@@ -376,6 +376,9 @@ fn register_active_hooks(supports_guard_gc_type: bool) {
         supports_guard_gc_type,
     });
     majit_gc::set_active_alloc_nursery_typed(Some(alloc_nursery_typed_via_active_runtime));
+    majit_gc::set_active_alloc_nursery_headerless_no_collect(Some(
+        alloc_nursery_headerless_no_collect_via_active_runtime,
+    ));
     majit_gc::set_active_alloc_nursery_typed_with_placement(Some(
         alloc_nursery_typed_with_placement_via_active_runtime,
     ));
@@ -1543,6 +1546,16 @@ fn alloc_nursery_typed_via_active_runtime(type_id: u32, size: usize) -> GcRef {
     with_cranelift_gc(|gc| gc.try_alloc_nursery_no_collect_typed(type_id, size)).unwrap_or(GcRef(0))
 }
 
+/// `majit_gc::AllocNurseryHeaderlessNoCollectFn` installed by
+/// `set_gc_allocator`. The metainterp's jitcode tracer allocates a `NEW` on a
+/// `headerless` descr through here so the object lands in the interpreter's
+/// own collected pool rather than the host heap, where its collector could not
+/// see it. Returns `GcRef(0)` when no GC is bound, leaving the caller on its
+/// own path.
+fn alloc_nursery_headerless_no_collect_via_active_runtime(size: usize) -> GcRef {
+    with_cranelift_gc(|gc| gc.alloc_nursery_headerless_no_collect(size)).unwrap_or(GcRef(0))
+}
+
 /// Placement-reporting companion of
 /// [`alloc_nursery_typed_via_active_runtime`].
 ///
@@ -11670,36 +11683,164 @@ impl CraneliftBackend {
                         &known_values,
                         op.arg(0).to_opref(),
                     );
-                    let result = emit_collecting_gc_call(
-                        &mut builder,
-                        ptr_type,
-                        call_conv,
-                        jf_ptr,
-                        &ref_root_slots,
-                        &defined_ref_vars,
-                        &stale_ref_vars,
-                        &demoted_failarg_slots,
-                        ref_root_base_ofs,
-                        per_call_gcmap,
-                        gc_alloc_nursery_headerless_shim as *const () as usize,
-                        &[size],
-                        Some(cl_types::I64),
-                    )
-                    .expect("headerless nursery allocation helper must return a value");
-                    jf_ptr = emit_reload_frame_if_necessary(&mut builder, ptr_type, call_conv);
-                    builder.ins().set_pinned_reg(jf_ptr);
-                    builder.def_var(var(vi), result);
-                    // malloc_cond parity: the headerless slow path
-                    // (gc_alloc_nursery_headerless_shim) returns NULL on
-                    // host/bounded out-of-memory; propagate before the
-                    // following stores dereference it.
-                    emit_memory_error_check(
-                        &mut builder,
-                        ptr_type,
-                        result,
-                        propagate_exception_descr_ptr,
-                        preamble_phase,
-                    );
+                    // x86/assembler.py:2556-2565 malloc_cond parity, headerless
+                    // variant — the same inline bump the `CallMallocNursery`
+                    // arm below emits and the dynasm backends already emit for
+                    // this opcode (`genop_call_malloc_nursery_headerless`).
+                    // Three differences from the headered shape: the bump is by
+                    // `size` alone (no `GcHeader::SIZE` reservation), no header
+                    // word is zeroed, and the result is the old nursery base
+                    // rather than `base + GcHeader::SIZE`.
+                    //
+                    // The raw bump is correct only while the active GC is
+                    // headerless-aware, which is the same invariant
+                    // `alloc_nursery_headerless`'s panicking default enforces on
+                    // the overflow path — whoever declares `headerless_structs`
+                    // upholds it. Without a bump surface (`nursery_free` /
+                    // `nursery_top` reported as 0) the op stays on the helper.
+                    let inline_bump = gc_nursery_addrs.filter(|&(nf, nt)| nf != 0 && nt != 0);
+                    if let Some((nf_addr, nt_addr)) = inline_bump {
+                        let flags = MemFlags::trusted();
+                        let nf_ptr = builder.ins().iconst(ptr_type, nf_addr as i64);
+                        let nt_ptr = builder.ins().iconst(ptr_type, nt_addr as i64);
+                        let free = builder.ins().load(ptr_type, flags, nf_ptr, 0);
+                        let new_free = builder.ins().iadd(free, size);
+                        let top = builder.ins().load(ptr_type, flags, nt_ptr, 0);
+                        // `nursery_top` is one-past-last, so the region is exhausted
+                        // exactly when the bumped free pointer runs past it — the
+                        // dynasm emitters' `b.hi` / `ja` slow-path edge.
+                        let fits =
+                            builder
+                                .ins()
+                                .icmp(IntCC::UnsignedLessThanOrEqual, new_free, top);
+
+                        // Same block-param carry as the headered arm: only the slow
+                        // path spills and reloads the ref roots, so the merge takes
+                        // every live ref as a parameter instead of letting the two
+                        // paths disagree on the variable's definition.
+                        let live_refs: Vec<(u32, usize)> = ref_root_slots
+                            .iter()
+                            .filter(|(var_idx, _)| defined_ref_vars.contains(var_idx))
+                            .copied()
+                            .collect();
+
+                        let fast_block = builder.create_block();
+                        let slow_block = builder.create_block();
+                        let merge_block = builder.create_block();
+                        builder.append_block_param(merge_block, ptr_type); // result
+                        builder.append_block_param(merge_block, ptr_type); // jf_ptr
+                        for _ in &live_refs {
+                            builder.append_block_param(merge_block, cl_types::I64);
+                        }
+                        builder.ins().brif(fits, fast_block, &[], slow_block, &[]);
+
+                        // fast: publish the bumped free pointer and hand back the
+                        // old base. Nothing here can collect, so no gcmap is pushed
+                        // and no ref root is spilled.
+                        builder.switch_to_block(fast_block);
+                        builder.seal_block(fast_block);
+                        builder.ins().store(flags, new_free, nf_ptr, 0);
+                        let mut fast_args: Vec<BlockArg> =
+                            vec![BlockArg::from(free), BlockArg::from(jf_ptr)];
+                        for &(var_idx, _) in &live_refs {
+                            fast_args.push(BlockArg::from(builder.use_var(var(var_idx))));
+                        }
+                        builder.ins().jump(merge_block, &fast_args);
+
+                        // slow: aarch64 `_build_malloc_slowpath` parity — spill the
+                        // ref roots and install the gcmap, then let the helper
+                        // collect and re-bump.
+                        builder.switch_to_block(slow_block);
+                        builder.seal_block(slow_block);
+                        builder.set_cold_block(slow_block);
+                        spill_ref_roots(
+                            &mut builder,
+                            jf_ptr,
+                            &ref_root_slots,
+                            &defined_ref_vars,
+                            &stale_ref_vars,
+                            &demoted_failarg_slots,
+                            ref_root_base_ofs,
+                        );
+                        emit_push_gcmap(&mut builder, jf_ptr, per_call_gcmap);
+                        let slow_r = emit_host_call(
+                            &mut builder,
+                            ptr_type,
+                            call_conv,
+                            gc_alloc_nursery_headerless_shim as *const () as usize,
+                            &[size],
+                            Some(cl_types::I64),
+                        )
+                        .expect("headerless nursery allocation helper must return a value");
+                        let jf_ptr_slow =
+                            emit_reload_frame_if_necessary(&mut builder, ptr_type, call_conv);
+                        emit_pop_gcmap(&mut builder, jf_ptr_slow, per_call_gcmap);
+                        reload_ref_roots(
+                            &mut builder,
+                            jf_ptr_slow,
+                            &ref_root_slots,
+                            &defined_ref_vars,
+                            &demoted_failarg_slots,
+                            ref_root_base_ofs,
+                        );
+                        let mut slow_args: Vec<BlockArg> =
+                            vec![BlockArg::from(slow_r), BlockArg::from(jf_ptr_slow)];
+                        for &(var_idx, _) in &live_refs {
+                            slow_args.push(BlockArg::from(builder.use_var(var(var_idx))));
+                        }
+                        builder.ins().jump(merge_block, &slow_args);
+
+                        builder.switch_to_block(merge_block);
+                        builder.seal_block(merge_block);
+                        let params = builder.block_params(merge_block).to_vec();
+                        let result = params[0];
+                        jf_ptr = params[1];
+                        builder.ins().set_pinned_reg(jf_ptr);
+                        for (i, &(var_idx, _)) in live_refs.iter().enumerate() {
+                            builder.def_var(var(var_idx), params[2 + i]);
+                        }
+                        builder.def_var(var(vi), result);
+                        // malloc_cond parity: the headerless slow path
+                        // (gc_alloc_nursery_headerless_shim) returns NULL on
+                        // host/bounded out-of-memory; propagate before the
+                        // following stores dereference it. The fast path cannot
+                        // produce NULL, but the check is on the merged value so a
+                        // slow-path NULL is caught on either edge.
+                        emit_memory_error_check(
+                            &mut builder,
+                            ptr_type,
+                            result,
+                            propagate_exception_descr_ptr,
+                            preamble_phase,
+                        );
+                    } else {
+                        let result = emit_collecting_gc_call(
+                            &mut builder,
+                            ptr_type,
+                            call_conv,
+                            jf_ptr,
+                            &ref_root_slots,
+                            &defined_ref_vars,
+                            &stale_ref_vars,
+                            &demoted_failarg_slots,
+                            ref_root_base_ofs,
+                            per_call_gcmap,
+                            gc_alloc_nursery_headerless_shim as *const () as usize,
+                            &[size],
+                            Some(cl_types::I64),
+                        )
+                        .expect("headerless nursery allocation helper must return a value");
+                        jf_ptr = emit_reload_frame_if_necessary(&mut builder, ptr_type, call_conv);
+                        builder.ins().set_pinned_reg(jf_ptr);
+                        builder.def_var(var(vi), result);
+                        emit_memory_error_check(
+                            &mut builder,
+                            ptr_type,
+                            result,
+                            propagate_exception_descr_ptr,
+                            preamble_phase,
+                        );
+                    }
                 }
                 OpCode::CallMallocNursery => {
                     // x86/assembler.py:2556-2565 malloc_cond parity.

majit/majit-backend-dynasm/src/runner.rs

@@ -180,6 +180,9 @@ fn register_active_hooks(supports_guard_gc_type: bool) {
         supports_guard_gc_type,
     });
     majit_gc::set_active_alloc_nursery_typed(Some(dynasm_alloc_nursery_typed));
+    majit_gc::set_active_alloc_nursery_headerless_no_collect(Some(
+        dynasm_alloc_nursery_headerless_no_collect,
+    ));
     majit_gc::set_active_alloc_nursery_typed_with_placement(Some(
         dynasm_alloc_nursery_typed_with_placement,
     ));
@@ -357,6 +360,26 @@ pub(crate) extern "C" fn dynasm_new_alloc(size: usize) -> *mut u8 {
     })
 }
 
+/// Headerless no-collect nursery trampoline for backend-agnostic callers.
+///
+/// The metainterp's jitcode tracer allocates a `NEW` on a `headerless` descr
+/// through here so the object lands in the interpreter's own collected pool
+/// rather than the host heap, where its collector could not see it. Returns
+/// null when no GC is bound, leaving the caller on its own path.
+fn dynasm_alloc_nursery_headerless_no_collect(size: usize) -> GcRef {
+    if let Some(r) = DYNASM_ACTIVE_GC.with(|c| {
+        c.borrow_mut()
+            .as_deref_mut()
+            .map(|g| g.alloc_nursery_headerless_no_collect(size))
+    }) {
+        return r;
+    }
+    if majit_gc::gc_sync::is_initialized() {
+        return majit_gc::gc_sync::gc_op(|g| g.alloc_nursery_headerless_no_collect(size));
+    }
+    GcRef::NULL
+}
+
 /// Host-side nursery allocation trampoline. Published via
 /// `majit_gc::set_active_alloc_nursery_typed` from `set_gc_allocator`
 /// so backend-agnostic callers (e.g. pyre-object `w_int_new`) can
@@ -465,6 +488,49 @@ fn dynasm_alloc_oldgen_typed(type_id: u32, size: usize) -> GcRef {
     majit_gc::gc_sync::gc_op(|g| g.alloc_oldgen_typed(type_id, size))
 }
 
+/// Allocate the struct a `bh_new` / `bh_new_with_vtable` descr describes
+/// (`llmodel.py:775-786`).
+///
+/// A GC-managed struct (real `type_id`) MUST be allocated through the GC so the
+/// collector can trace its pointer fields: a resume-materialized virtual (e.g.
+/// an inlined-callee `PyFrame`) holds a `locals_cells_stack` ref to its arrays,
+/// and a raw `libc::malloc` block is invisible to the GC, so a minor collection
+/// during the blackhole forward run frees those arrays out from under the
+/// frame.  Allocate in the non-moving old generation (mark-sweep), mirroring
+/// `w_int_new`/`w_float_new`: the blackhole register file and the deep forward
+/// recursion capture raw pointers to the materialized struct that the resume
+/// path does not re-root across the minor collections it triggers, so a moving
+/// nursery object would leave those captures stale.  Old-gen keeps every
+/// materialized pointer stable for the lifetime of the resume.
+///
+/// A headerless struct lives in the interpreter's own `headerless_structs` pool
+/// and carries no `type_id` word at `ref - 8`, so it takes the headerless
+/// nursery allocator instead: `alloc_oldgen_typed` returns
+/// `base + GcHeader::SIZE`, which would shift every field offset the descr
+/// carries.
+///
+/// Non-GC descrs (`type_id == 0`, raw buffers) and a runtime with no allocator
+/// hook installed (unit tests) keep the plain zeroed malloc.
+fn bh_alloc_struct(sizedescr: &majit_translate::jitcode::BhDescr) -> *mut libc::c_void {
+    let size = sizedescr.as_size();
+    let gc_ptr = if sizedescr.is_headerless() {
+        majit_gc::alloc_nursery_headerless_no_collect(size).0
+    } else {
+        match sizedescr.resolve_gc_tid() {
+            0 => 0,
+            type_id => dynasm_alloc_oldgen_typed(type_id, size).0,
+        }
+    };
+    if gc_ptr != 0 {
+        return gc_ptr as *mut libc::c_void;
+    }
+    let ptr = unsafe { libc::malloc(size) };
+    if !ptr.is_null() {
+        unsafe { libc::memset(ptr, 0, size) };
+    }
+    ptr
+}
+
 /// User-level `gc.collect()` trampoline — drives `GcAllocator::collect_full`
 /// on the active dynasm-owned GC. PyPy's `pypy/module/gc/interp_gc.py:7-26`
 /// runs `rgc.collect()` from app-level `gc.collect`; this is the dynasm
@@ -2965,40 +3031,12 @@ impl Backend for DynasmBackend {
     }
 
     fn bh_new(&self, sizedescr: &majit_translate::jitcode::BhDescr) -> i64 {
-        let size = sizedescr.as_size();
-        let ptr = unsafe { libc::malloc(size) };
-        if !ptr.is_null() {
-            unsafe { libc::memset(ptr, 0, size) };
-        }
-        ptr as i64
+        bh_alloc_struct(sizedescr) as i64
     }
 
     fn bh_new_with_vtable(&self, sizedescr: &majit_translate::jitcode::BhDescr) -> i64 {
-        let size = sizedescr.as_size();
         let vtable = sizedescr.get_vtable();
-        // A GC-managed struct (real `type_id`) MUST be allocated through the GC
-        // so the collector can trace its pointer fields: a resume-materialized
-        // virtual (e.g. an inlined-callee `PyFrame`) holds a `locals_cells_stack`
-        // ref to its arrays, and a raw `libc::malloc` block is invisible to the
-        // GC, so a minor collection during the blackhole forward run frees those
-        // arrays out from under the frame.  Allocate in the non-moving old
-        // generation (mark-sweep), mirroring `w_int_new`/`w_float_new`: the
-        // blackhole register file and the deep forward recursion capture raw
-        // pointers to the materialized struct that the resume path does not
-        // re-root across the minor collections it triggers, so a moving nursery
-        // object would leave those captures stale.  Old-gen keeps every
-        // materialized pointer stable for the lifetime of the resume.  Non-GC
-        // descrs (`type_id == 0`, raw buffers) keep the plain malloc.
-        let type_id = sizedescr.resolve_gc_tid();
-        let ptr = if type_id != 0 {
-            dynasm_alloc_oldgen_typed(type_id, size).0 as *mut libc::c_void
-        } else {
-            let ptr = unsafe { libc::malloc(size) };
-            if !ptr.is_null() {
-                unsafe { libc::memset(ptr, 0, size) };
-            }
-            ptr
-        };
+        let ptr = bh_alloc_struct(sizedescr);
         if !ptr.is_null() {
             unsafe {
                 // llmodel.py:780-782: if self.vtable_offset is not None: