diff --git a/.github/workflows/pyre-ci.yml b/.github/workflows/pyre-ci.yml index 95c21dd90d1..eb111445e0c 100644 --- a/.github/workflows/pyre-ci.yml +++ b/.github/workflows/pyre-ci.yml @@ -67,6 +67,10 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: cache-bin: false + # Restore-only on PRs (they pull main's cache via restore-keys); save + # only on main so per-PR target caches don't multiply across refs past + # GitHub's 10 GB cache budget and trigger LRU eviction. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Cache Charon binary id: charon-cache uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 @@ -200,6 +204,10 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: cache-bin: false + # Restore-only on PRs (they pull main's cache via restore-keys); save + # only on main so per-PR target caches don't multiply across refs past + # GitHub's 10 GB cache budget and trigger LRU eviction. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Download Charon artifact # Run-scoped handoff from prepare-charon-llbc. Cross-run reuse still # comes from the prepare job's cache; consumers avoid cache eviction @@ -309,6 +317,10 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: cache-bin: false + # Restore-only on PRs (they pull main's cache via restore-keys); save + # only on main so per-PR target caches don't multiply across refs past + # GitHub's 10 GB cache budget and trigger LRU eviction. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Download Charon artifact # Run-scoped handoff from prepare-charon-llbc. Cross-run reuse still # comes from the prepare job's cache; consumers avoid cache eviction @@ -334,6 +346,14 @@ jobs: test -s build/llbc/pyre-object.ullbc test -s build/llbc/pyre-interpreter.ullbc test -s build/llbc/pyre-jit.ullbc + - name: Add wasm32 target (Linux only) + # check.py adds wasm to its default backends when this target is + # installed, so this is what makes the Linux leg build and run the wasm + # backend too. wasm output is platform-independent, so exercising it on + # one OS is enough — macOS/Windows keep no wasm32 target and stay on the + # native backends. + if: runner.os == 'Linux' + run: rustup target add wasm32-unknown-unknown - name: Run pyre/check.py env: PYRE_CHECK_PYTHON3: ${{ steps.cpython.outputs.python-path }} @@ -386,6 +406,10 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: cache-bin: false + # Restore-only on PRs (they pull main's cache via restore-keys); save + # only on main so per-PR target caches don't multiply across refs past + # GitHub's 10 GB cache budget and trigger LRU eviction. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Download Charon artifact # Run-scoped handoff from prepare-charon-llbc. Cross-run reuse still # comes from the prepare job's cache; consumers avoid cache eviction @@ -444,6 +468,10 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: cache-bin: false + # Restore-only on PRs (they pull main's cache via restore-keys); save + # only on main so per-PR target caches don't multiply across refs past + # GitHub's 10 GB cache budget and trigger LRU eviction. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Download Charon artifact # Run-scoped handoff from prepare-charon-llbc. Cross-run reuse still # comes from the prepare job's cache; consumers avoid cache eviction diff --git a/majit/majit-backend-wasm/src/codegen.rs b/majit/majit-backend-wasm/src/codegen.rs index 8592f512b88..a6120ce277c 100644 --- a/majit/majit-backend-wasm/src/codegen.rs +++ b/majit/majit-backend-wasm/src/codegen.rs @@ -313,18 +313,55 @@ fn has_ref_store_op(ops: &[Op], ref_homes: &RefHomes) -> bool { .any(|op| write_barrier_base(op, ref_homes).is_some()) } -/// Emit a write-barrier trampoline call on `base_ref` before a ref-storing -/// field/array store. The host helper `wasm_jit_write_barrier` checks -/// TRACK_YOUNG_PTRS and remembers an old→young store, standing in for the -/// `COND_CALL_GC_WB` the native GC rewrite pass inserts. Operand-stack-neutral: -/// every push is consumed by a store or the call. +/// Emit a write-barrier check on `base_ref` before a ref-storing field/array +/// store, standing in for the `COND_CALL_GC_WB` the native GC rewrite pass +/// inserts. When the residual type family is declared (`residual_type_base`), +/// the TRACK_YOUNG_PTRS flag is tested INLINE (assembler.py:2382 +/// `genop_discard_cond_call_gc_wb` — test the header flag byte, jump over the +/// slow call when clear) and only a flagged old object takes the +/// `wasm_jit_write_barrier` `call_indirect`; a young or already-remembered +/// base skips the helper entirely. Otherwise the unconditional helper routes +/// through the `jit_call` host trampoline (the helper re-checks the flag). +/// Operand-stack-neutral: every push is consumed by a store, the call, or +/// the result drop. fn emit_write_barrier( sink: &mut InstructionSink<'_>, constants: &majit_ir::VecMap, - jit_call: u32, + jit_call_idx: Option, + residual_type_base: Option, wb_fn_ptr: i64, base_ref: OpRef, ) { + if let Some(base) = residual_type_base { + // Header word is a u64 at `obj - GcHeader::SIZE` with the flags in + // its upper half (`FLAG_SHIFT == 32`), so on little-endian wasm32 the + // flags live in the i32 at `obj - 4`; TRACK_YOUNG_PTRS is flag bit 0. + const FLAGS_HALF_BACKOFS: i32 = (majit_gc::header::GcHeader::SIZE / 2) as i32; + const WB_FLAG: i32 = majit_gc::flags::TRACK_YOUNG_PTRS as i32; + const _: () = assert!(majit_gc::header::FLAG_SHIFT == 32); + const _: () = assert!(majit_gc::flags::TRACK_YOUNG_PTRS <= u32::MAX as u64); + emit_resolve(sink, constants, base_ref); + sink.i32_wrap_i64(); + sink.i32_const(FLAGS_HALF_BACKOFS); + sink.i32_sub(); + sink.i32_load(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + sink.i32_const(WB_FLAG); + sink.i32_and(); + sink.if_(BlockType::Empty); + emit_resolve(sink, constants, base_ref); + sink.i32_const(wb_fn_ptr as i32); + sink.call_indirect(0, base + 1); + sink.drop(); // returns 0; ignored + sink.end(); + return; + } + let Some(jit_call) = jit_call_idx else { + return; + }; // func_ptr = wasm_jit_write_barrier sink.local_get(0); sink.i64_const(wb_fn_ptr); @@ -479,6 +516,62 @@ fn residual_call_i64_arity(op: &Op) -> Option { Some(nargs) } +/// Void-recorded counterpart of [`residual_call_i64_arity`]: an eligible +/// void residual CALL whose descr records the dummy-word C ABI +/// (`result_size == 8`, minted by `make_call_descr_void_word_abi`) — the +/// callee is really `(i64×n) -> i64` with the result ignored, so it lowers +/// through the same i64 type family with a trailing `drop`. A plain void +/// descr (`result_size == 0`) may target a genuinely `()`-returning callee +/// OR a word-returning one (the reflective host trampoline absorbs the +/// difference), so it stays on `jit_call`. Same force/GIL/cond exclusions +/// as the i64 family. +fn residual_call_void_word_arity(op: &Op) -> Option { + use OpCode::*; + if !matches!(op.opcode, CallN | CallPureN | CallLoopinvariantN) { + return None; + } + let descr = op.getdescr()?; + let cd = descr.as_call_descr()?; + if cd.result_type() != Type::Void || cd.result_size() != 8 { + return None; + } + let arg_types = cd.arg_types(); + if arg_types + .iter() + .any(|t| !matches!(t, Type::Int | Type::Ref)) + { + return None; + } + let nargs = op.getarglist().len().saturating_sub(1); + if arg_types.len() != nargs { + return None; + } + Some(nargs) +} + +/// Arity of `op`'s in-module `(i64×n) -> i64` lowering, if it has one: an +/// eligible residual CALL (word-result or word-ABI void), a `New*` +/// allocation (the `wasm_jit_alloc*` helper targets are plain +/// `extern "C" fn(i64×n) -> i64` table entries), or a ref-storing store +/// (its `wasm_jit_write_barrier` helper takes 1 arg). All of these share +/// the residual-call type family, so one max covers them. +fn direct_helper_i64_arity(op: &Op, ref_homes: &RefHomes) -> Option { + if let Some(n) = residual_call_i64_arity(op) { + return Some(n); + } + if let Some(n) = residual_call_void_word_arity(op) { + return Some(n); + } + match op.opcode { + // wasm_jit_alloc(type_id, size) + OpCode::New | OpCode::NewWithVtable => Some(2), + // wasm_jit_alloc_array(type_id, base_size, item_size, length, len_offset) + OpCode::NewArray | OpCode::NewArrayClear => Some(5), + // wasm_jit_write_barrier(base) + _ => write_barrier_base(op, ref_homes).map(|_| 1), + } +} + fn has_call_ops(ops: &[Op]) -> bool { // Allocation ops (`New*`, `Newstr`/`Newunicode`) also reach the host via // the `jit_call` trampoline, so the import must be present for them too. @@ -609,9 +702,10 @@ pub struct CaParams { /// `fail_descrs`. pub source_compiled_ptr: u64, /// `__indirect_function_table` slot (`fn as usize`) of - /// `lib.rs::wasm_jit_ca_alloc_frame`. The CA arm routes through the - /// `jit_call` trampoline to allocate each callee frame as a GC-managed - /// old-gen `JitFrame` (push_jf-rooted, traced by its own per-frame gcmap). + /// `lib.rs::wasm_jit_ca_alloc_frame`, which allocates each callee frame as + /// a GC-managed old-gen `JitFrame` (push_jf-rooted, traced by its own + /// per-frame gcmap). `call_indirect`ed in-module through the residual + /// `(i64,i64)->i64` type when declared, else via the `jit_call` trampoline. pub ca_alloc_fn_ptr: i64, /// `__indirect_function_table` slot of `lib.rs::wasm_jit_ca_pop_frame`, /// called on CA-arm exit to pop the callee frame off the jitframe shadow @@ -623,6 +717,25 @@ pub struct CaParams { pub callee_gcmap_ptr: i64, } +/// Inline nursery-bump fast-path parameters for `New`/`NewWithVtable` +/// (rewrite.py's malloc fast path over the gc.py:525-531 +/// `get_nursery_free_addr`/`get_nursery_top_addr` surface, which the x86 +/// backend lowers as `malloc_cond`: load free, bump, compare top, call the +/// slow path only on overflow). `None` keeps every allocation on the +/// `wasm_jit_alloc` helper call. +pub struct NurseryAllocParams { + /// Linear-memory address of the GC's `nursery_free` bump pointer. + pub free_addr: u32, + /// Linear-memory address of the GC's `nursery_top` limit pointer. + pub top_addr: u32, + /// `max_nursery_object_size` — a total size above this allocates in + /// old-gen, so the inline path only applies below it. + pub large_threshold: usize, + /// Type ids whose allocation is a plain bump + header write (no + /// destructor / weakref side-list registration). + pub plain_tids: std::collections::HashSet, +} + /// Build a wasm module from majit IR. pub fn build_wasm_module( inputargs: &[InputArg], @@ -634,7 +747,14 @@ pub fn build_wasm_module( alloc_fn_ptr: i64, alloc_array_fn_ptr: i64, wb_fn_ptr: i64, + // Inline nursery-bump fast path for eligible `New`/`NewWithVtable` + // (see `NurseryAllocParams`); `None` keeps allocations on the helper. + nursery: Option<&NurseryAllocParams>, fail_index_base: u32, + // Whether this trace is compiled as a LOOP (`compile_loop`) rather than a + // bridge. The base alone cannot tell them apart: both draw it from the + // global fail-index space (`failguard::fail_descr_base`). + is_loop: bool, // Table slot of the loop a JUMP-with-no-local-LABEL re-enters (a loop-closing // bridge). `0` for a loop trace (its JUMP is a local back-edge `br`) and for a // straight-line bridge (no JUMP). When set, the terminal external JUMP writes @@ -642,18 +762,22 @@ pub fn build_wasm_module( // loop's table slot — a wasm tail call, so the loop⇄bridge cycle runs at // constant stack depth instead of growing one frame per iteration. external_jump_slot: u32, + // Resume-at-LABEL dispatch key for the terminal external JUMP (`target + // label ordinal + 1`, or 0 for a non-peeled target); see `build_function`. + external_jump_key: u32, // Self-recursive CALL_ASSEMBLER arm parameters (`PYRE_WASM_CA`); `emit_ca` // off keeps the module byte-identical. ca: CaParams, ) -> Result<(Vec, Vec, usize, u32, Option>), BackendError> { let (mut guards, num_vars) = collect_guards_and_vars(inputargs, ops); - // A bridge's guard/finish exits share one fail-index namespace with the - // source loop they attach to: their descrs are appended to the source - // loop's `fail_descrs` (so `execute_token` resolves `fail_descrs[frame[0]]` - // uniformly), and `frame[0]` carries the global index. `build_function` - // seeds its `guard_idx` counter with this base so each exit writes - // `base + local`; mirror that here on the returned `GuardExit.fail_index`. + // Every trace's guard/finish exits draw their indices from ONE global + // fail-index space (`failguard::FAIL_DESCR_REGISTRY`): a cross-trace chain + // can exit through a sibling loop's guard, so `frame[0]` must be + // resolvable without knowing which chained module wrote it. + // `build_function` seeds its `guard_idx` counter with this base so each + // exit writes `base + local`; mirror that here on the returned + // `GuardExit.fail_index`. for g in &mut guards { g.fail_index += fail_index_base; } @@ -668,14 +792,17 @@ pub fn build_wasm_module( // trace reads it and `compile_bridge` (guest-side) writes it. On native // builds the trace is never executed, so `alloc_bridge_cells` returns 0 and // the dispatch is omitted entirely — the module stays byte-identical. - // A loop-closing bridge needs a `Label` to chain into; the - // self-recursive CALL_ASSEMBLER case (`PYRE_WASM_CA`) chains a guard exit of - // a Label-less recursion loop (`fail_index_base == 0` ⇒ a loop, not a - // bridge) into its CA bridge, so allocate cells for that too. Gated on the - // flag, so flag-off stays byte-identical. + // Label-less traces that still want guard cells: the self-recursive + // CALL_ASSEMBLER case (`PYRE_WASM_CA`) chains a guard exit of a Label-less + // recursion LOOP (`is_loop`, not a bridge) into its CA bridge; and with + // bridge chaining on, a BRIDGE's own guards chain nested sub-bridges the + // same way (a hot guard inside a chained bridge would otherwise round-trip + // to the host forever). Both gated on their runtime flags, so flag-off + // stays byte-identical. let want_dispatch = !guards.is_empty() && (ops.iter().any(|op| op.opcode == OpCode::Label) - || (fail_index_base == 0 && crate::wasm_ca_enabled())); + || (is_loop && crate::wasm_ca_enabled()) + || (!is_loop && crate::wasm_bridges_enabled())); let (cells_base, cells_owner) = if want_dispatch { alloc_bridge_cells(guards.len()) } else { @@ -740,11 +867,24 @@ pub fn build_wasm_module( let needs_table = needs_call || bridge_dispatch || ca.emit_ca; // In-module residual calls (`WASM_DIRECT_RESIDUAL_CALL`): the largest - // eligible `(i64×n)->i64` residual-call arity in this trace, or `None` if - // there are none. Each distinct arity `0..=max` gets its own function type - // (declared below) so the CALL arm can `call_indirect` with a static type. + // eligible `(i64×n)->i64` arity in this trace — residual CALLs (word + // result or word-ABI void) plus the `New*` / write-barrier helper + // targets, which share the same uniform-i64 ABI — or `None` if there + // are none. Each distinct arity `0..=max` gets its own function type + // (declared below) so those arms can `call_indirect` with a static type. let residual_max_arity = if WASM_DIRECT_RESIDUAL_CALL { - ops.iter().filter_map(residual_call_i64_arity).max() + let scanned = ops + .iter() + .filter_map(|op| direct_helper_i64_arity(op, &ref_homes)) + .max(); + if ca.emit_ca { + // The CA arm's frame helpers (`wasm_jit_ca_alloc_frame(frame_bytes, + // gcmap_ptr)` / `wasm_jit_ca_pop_frame(frame_base)`) lower through + // this same `(i64×n)->i64` family; make sure arity 2 is declared. + Some(scanned.map_or(2, |m| m.max(2))) + } else { + scanned + } } else { None }; @@ -846,11 +986,13 @@ pub fn build_wasm_module( alloc_fn_ptr, alloc_array_fn_ptr, wb_fn_ptr, + nursery, &ref_homes, cells_base, bridge_dispatch, fail_index_base, external_jump_slot, + external_jump_key, residual_max_arity.map(|_| residual_type_base), ca, bridge_finish_fi, @@ -881,14 +1023,21 @@ fn build_function( alloc_fn_ptr: i64, alloc_array_fn_ptr: i64, wb_fn_ptr: i64, + nursery: Option<&NurseryAllocParams>, ref_homes: &RefHomes, cells_base: u32, bridge_dispatch: bool, fail_index_base: u32, external_jump_slot: u32, + // Resume-at-LABEL dispatch key the terminal external JUMP writes before + // tail-calling `external_jump_slot`: `target label ordinal + 1`, so the + // target's entry `br_table` lands on that label's resume loader. `0` when + // the target is not peeled (no dispatch reads the slot). + external_jump_key: u32, // Base wasm type index of the `(i64×n)->i64` residual-call types (type // `residual_type_base + n` for arity `n`), or `None` when the trace has no - // eligible residual call so the CALL arm always uses the `jit_call` path. + // eligible residual call / `New*` / write barrier, so those arms always + // use the `jit_call` path. residual_type_base: Option, // Self-recursive CALL_ASSEMBLER arm (`PYRE_WASM_CA`). `ca.emit_ca` off keeps // the body byte-identical. @@ -913,9 +1062,13 @@ fn build_function( // flag-off module keeps exactly one i32 local (byte-identical). let ca_cfp_local = num_vars + UMULHI_SCRATCH + 2; let ca_fi_local = num_vars + UMULHI_SCRATCH + 3; + // One more i32 scratch when the inline nursery-bump fast path is armed: + // it holds the loaded `nursery_free` across the bump/commit sequence. + let base_i32_locals: u32 = if ca.emit_ca { 3 } else { 1 }; + let alloc_scratch_local = num_vars + UMULHI_SCRATCH + 1 + base_i32_locals; let mut func = Function::new(vec![ (num_vars + UMULHI_SCRATCH, ValType::I64), - (if ca.emit_ca { 3 } else { 1 }, ValType::I32), + (base_i32_locals + nursery.is_some() as u32, ValType::I32), ]); let mut sink = func.instructions(); @@ -937,37 +1090,61 @@ fn build_function( let loop_label_idx = ops.iter().rposition(|op| op.opcode == OpCode::Label); let has_loop = loop_label_idx.is_some(); - // A Label-less recursion loop with bridge dispatch (`PYRE_WASM_CA`): there is - // no `loop`, but its guard/Finish exits still need to `br` to the function - // epilogue so the epilogue's cell dispatch can chain a failing guard into its - // CA bridge (instead of each guard early-returning to the host). Wrap the - // body in one exit `block` and route exits through it, exactly as a loop - // does. Only loops reach here (`bridge_dispatch` is gated on - // `fail_index_base == 0` for the Label-less case), so a CA bridge — itself - // Label-less — keeps its byte-identical straight-line layout. - let ca_straightline_dispatch = !has_loop && bridge_dispatch; + // A Label-less trace with bridge dispatch — a `PYRE_WASM_CA` recursion + // loop, or (chaining on) a bridge whose own guards chain nested + // sub-bridges: there is no `loop`, but its guard/Finish exits still need + // to `br` to the function epilogue so the epilogue's cell dispatch can + // chain a failing guard in-module (instead of each guard early-returning + // to the host). Wrap the body in one exit `block` and route exits through + // it, exactly as a loop does. A loop-closing bridge's terminal external + // JUMP is unaffected — `return_call_indirect` leaves the function from + // inside the block. + let straightline_dispatch = !has_loop && bridge_dispatch; // Resume-at-LABEL: a peeled loop wraps its preamble in a dispatch so a - // loop-closing bridge can re-enter AT the (last) LABEL — where the `loop` - // is — skipping the preamble, in-module instead of round-tripping through - // the host. Keyed on the peeled shape (single- OR multi-label); every other - // trace (non-peeled loop, straight-line, bridge) keeps its byte-identical - // layout. The dispatch resumes only at the LAST label, so the wrapper is - // byte-identical whether the source is single- or multi-label — the - // multi-label-but-non-last-label case is declined in `compile_bridge`. + // loop-closing bridge can re-enter AT any LABEL — key = label ordinal + 1 + // — skipping the code before it, in-module instead of round-tripping + // through the host. Keyed on the peeled shape (single- OR multi-label); + // every other trace (non-peeled loop, straight-line, bridge) keeps its + // byte-identical layout. Each label gets a (past_loader, loader) block + // pair; the entry `br_table` jumps to the keyed label's resume loader, + // and the fall-through path `br`s over each loader. Key 0 (and any + // out-of-range key) runs the function from its entry (the preamble). let key_dispatch = is_resumable_peeled(ops); + let num_labels = ops.iter().filter(|op| op.opcode == OpCode::Label).count(); + let all_label_args: Vec> = if key_dispatch { + ops.iter() + .filter(|op| op.opcode == OpCode::Label) + .map(|op| op.getarglist().iter().map(|a| a.to_opref()).collect()) + .collect() + } else { + Vec::new() + }; if key_dispatch { // block $exit (A) — guard/Finish exits br here -> epilogue. - // block $past_loader (B) — the preamble path br's over the resume loader. - // block $skip_preamble (C) — a resuming bridge br's out of here, past the - // preamble + entry loader, landing in the resume loader. + // Per label j (opened outermost = last label): + // block $past_loader_j (B_j) — the fall-through path br's over the + // label-j resume loader. + // block $loader_j (C_j) — the `br_table` lands here (its end) for + // key j+1: the label-j resume loader. + // block $dispatch (D) — key 0 br's here: run from the entry. sink.block(BlockType::Empty); // A $exit - sink.block(BlockType::Empty); // B $past_loader - sink.block(BlockType::Empty); // C $skip_preamble + for _ in 0..num_labels { + sink.block(BlockType::Empty); // B_j (j descending) + sink.block(BlockType::Empty); // C_j + } + sink.block(BlockType::Empty); // D $dispatch sink.local_get(0); sink.i64_load(mem64(DISPATCH_KEY_OFS)); sink.i32_wrap_i64(); - sink.br_if(0); // key != 0 -> resume: skip the preamble + entry loader + // Depths at this point, innermost first: D=0, then (C_j, B_j) pairs + // with C_j at 2j+1. Entry j+1 of the table targets C_j; entry 0 and + // the default target D (the entry path). + let br_targets: Vec = std::iter::once(0) + .chain((0..num_labels as u32).map(|j| 2 * j + 1)) + .collect(); + sink.br_table(br_targets, 0); + sink.end(); // end D $dispatch — key-0 entry path continues here } // Load inputs from frame into locals, and store Ref inputs to their homes. @@ -996,62 +1173,70 @@ fn build_function( } // Non-key_dispatch loop: the single exit block A (preamble + body share it). - // key_dispatch already opened A/B/C above. A Label-less CA dispatch loop also - // opens A so its guard/Finish exits `br` out to the epilogue. - if (has_loop || ca_straightline_dispatch) && !key_dispatch { + // key_dispatch already opened A/B/C above. A Label-less dispatch trace (CA + // loop, or a bridge chaining nested sub-bridges) also opens A so its + // guard/Finish exits `br` out to the epilogue. + if (has_loop || straightline_dispatch) && !key_dispatch { sink.block(BlockType::Empty); } // Seed with the fail-index base so each guard/finish exit writes - // `base + local` into `frame[0]` (loops pass 0; bridges pass the source - // loop's descr count so their indices land past the loop's). The local + // `base + local` into `frame[0]` (every trace passes the next free index + // of the global fail-index space, `failguard::fail_descr_base`). The local // `guard_idx` counter and `collect_guards_and_vars`'s `fail_index` counter // increment in lockstep over the same ops, so the value written matches the // returned `GuardExit.fail_index` (also offset by the base). let mut guard_idx = fail_index_base; let mut in_loop_body = false; + let mut labels_passed = 0usize; for (op_idx, op) in ops.iter().enumerate() { - if Some(op_idx) == loop_label_idx { - if key_dispatch { - // End of the preamble (key-0 path). Branch over the resume - // loader to the loop, then close C, emit the loader (resume - // path only), close B, and open the loop. From inside C, `br 1` - // targets B's end (just before the loop), skipping the loader. - sink.br(1); // preamble done -> past_loader, over the resume loader - sink.end(); // end C $skip_preamble (resume path lands here) - // Resume loader: a loop-closing bridge wrote each label arg into - // frame slot i (positionally, matching the in-loop JUMP move); - // load them into the label-arg locals and refresh their Ref - // homes, mirroring the JUMP's ref-home refresh below. The - // preamble path skipped this via the `br 1` above. - let label_args = find_label_args(ops); - for (i, la) in label_args.iter().enumerate() { + if op.opcode == OpCode::Label && key_dispatch { + // End of the segment before label j (key-0 / earlier-label path). + // Branch over the resume loader, then close C_j, emit the loader + // (resume path only), and close B_j. From inside C_j, `br 1` + // targets B_j's end, skipping the loader. + sink.br(1); // segment done -> past_loader_j, over the resume loader + sink.end(); // end C_j (the br_table lands here for key j+1) + // Resume loader: a loop-closing bridge wrote each label arg into + // frame slot i (positionally, matching the in-loop JUMP move); + // load them into the label-arg locals and refresh their Ref + // homes, mirroring the JUMP's ref-home refresh below. The + // fall-through path skipped this via the `br 1` above. + for (i, la) in all_label_args[labels_passed].iter().enumerate() { + sink.local_get(0); + sink.i64_load(mem64(FRAME_SLOT_BASE + i as u64 * SLOT_SIZE)); + sink.local_set(1 + la.raw()); + if let Some(h) = ref_homes.home(*la) { sink.local_get(0); - sink.i64_load(mem64(FRAME_SLOT_BASE + i as u64 * SLOT_SIZE)); - sink.local_set(1 + la.raw()); - if let Some(h) = ref_homes.home(*la) { - sink.local_get(0); - sink.local_get(1 + la.raw()); - sink.i64_store(mem64(HOME_SLOT_BASE + h as u64 * SLOT_SIZE)); - } + sink.local_get(1 + la.raw()); + sink.i64_store(mem64(HOME_SLOT_BASE + h as u64 * SLOT_SIZE)); } - sink.end(); // end B $past_loader } + sink.end(); // end B_j $past_loader + labels_passed += 1; + } + if Some(op_idx) == loop_label_idx { sink.loop_(BlockType::Empty); in_loop_body = true; } // Depth (from statement level) of the enclosing `block` that guard // exits `br` to. Without `key_dispatch`: preamble = 0, loop body = 1 // (the `loop` sits between the body and block A). With `key_dispatch` - // the preamble sits two blocks deeper (inside B and C), so it br's to - // depth 2; the body is unchanged at 1 (B and C close before the loop). - // `None` for straight-line traces (no block emitted). + // a segment that still has `num_labels - labels_passed` labels ahead + // sits inside that many (B_j, C_j) pairs, so it br's to depth + // `2 * remaining`; the body is unchanged at 1 (every pair closes + // before the loop). `None` for straight-line traces (no block + // emitted). let block_exit_depth = match (has_loop, in_loop_body) { - // Label-less CA dispatch loop: one exit block A (depth 0), no `loop`. - (false, _) if ca_straightline_dispatch => Some(0u32), + // Label-less dispatch trace: one exit block A (depth 0), no `loop`. + (false, _) if straightline_dispatch => Some(0u32), (false, _) => None, - (true, false) => Some(if key_dispatch { 2u32 } else { 0u32 }), + (true, false) => Some(if key_dispatch { + 2 * (num_labels - labels_passed) as u32 + } else { + 0u32 + }), (true, true) => Some(1u32), }; match op.opcode { @@ -1079,15 +1264,15 @@ fn build_function( sink.i64_store(mem64(FRAME_SLOT_BASE + i as u64 * SLOT_SIZE)); } // Set the resume-at-LABEL dispatch key so a peeled target - // re-enters at its (last) LABEL — skipping the preamble — instead - // of re-running it from the function entry. Harmless for a - // non-peeled target, which has no dispatch and ignores the slot. - // `compile_bridge` only compiles this bridge when its JUMP - // resumes at the target's last label (always so for a single- - // label source; checked via the JUMP descr for a multi-label - // one), so key 1 always lands at the `loop`. + // re-enters at the JUMP's target LABEL — skipping the code + // before it — instead of re-running the function from its + // entry. `compile_bridge` resolves the target label ordinal + // from the JUMP descr and passes `ordinal + 1` here; the + // target's entry `br_table` lands on that label's resume + // loader. Harmless for a non-peeled target, which has no + // dispatch and ignores the slot (`external_jump_key` 0). sink.local_get(0); // frame_ptr - sink.i64_const(1); // dispatch key = resume at LABEL + sink.i64_const(external_jump_key as i64); // dispatch key sink.i64_store(mem64(DISPATCH_KEY_OFS)); sink.local_get(0); // frame_ptr argument to the loop sink.i32_const(external_jump_slot as i32); // table slot @@ -1515,10 +1700,15 @@ fn build_function( } } OpCode::SetfieldGc | OpCode::SetfieldRaw => { - if let (Some(jit_call), Some(base)) = - (jit_call_idx, write_barrier_base(op, ref_homes)) - { - emit_write_barrier(&mut sink, constants, jit_call, wb_fn_ptr, base); + if let Some(base) = write_barrier_base(op, ref_homes) { + emit_write_barrier( + &mut sink, + constants, + jit_call_idx, + residual_type_base, + wb_fn_ptr, + base, + ); } emit_resolve(&mut sink, constants, op.arg(0).to_opref()); // struct ptr sink.i32_wrap_i64(); @@ -1599,10 +1789,15 @@ fn build_function( } } OpCode::SetarrayitemGc | OpCode::SetarrayitemRaw => { - if let (Some(jit_call), Some(base)) = - (jit_call_idx, write_barrier_base(op, ref_homes)) - { - emit_write_barrier(&mut sink, constants, jit_call, wb_fn_ptr, base); + if let Some(base) = write_barrier_base(op, ref_homes) { + emit_write_barrier( + &mut sink, + constants, + jit_call_idx, + residual_type_base, + wb_fn_ptr, + base, + ); } emit_array_addr(&mut sink, constants, op); emit_resolve(&mut sink, constants, op.arg(2).to_opref()); // value @@ -1644,10 +1839,15 @@ fn build_function( } } OpCode::SetinteriorfieldGc => { - if let (Some(jit_call), Some(base)) = - (jit_call_idx, write_barrier_base(op, ref_homes)) - { - emit_write_barrier(&mut sink, constants, jit_call, wb_fn_ptr, base); + if let Some(base) = write_barrier_base(op, ref_homes) { + emit_write_barrier( + &mut sink, + constants, + jit_call_idx, + residual_type_base, + wb_fn_ptr, + base, + ); } emit_resolve(&mut sink, constants, op.arg(0).to_opref()); sink.i32_wrap_i64(); @@ -2066,35 +2266,44 @@ fn build_function( // Refs still hold pre-call (from-space) addresses on return, so // reload them from the (forwarded) homes after the call. OpCode::CallAssemblerR if ca.emit_ca => { - let jit_call = - jit_call_idx.expect("CA arm needs jit_call for the frame trampolines"); let vi = op.pos.get().raw(); // `external_jump_slot` is the source loop's table slot (the CA // self-target), plumbed by `compile_bridge`. let self_slot = external_jump_slot as i32; - // Allocate the callee frame as a GC JitFrame via the jit_call - // trampoline (`wasm_jit_ca_alloc_frame(frame_bytes, gcmap_ptr)`); - // the call slots live in THIS (caller) frame at local 0. + // Allocate the callee frame as a GC JitFrame + // (`wasm_jit_ca_alloc_frame(frame_bytes, gcmap_ptr)` — a plain + // `(i64,i64)->i64` table entry that never itself collects, so + // it lowers like an eligible residual call when the type family + // is declared; otherwise via the jit_call trampoline). // `ca_cfp_local = frame_base + FIRST_ITEM_OFFSET` is the // bespoke-layout frame pointer — every `mem64(OFS)` below is // relative to it, exactly as the source loop reads its local 0. - sink.local_get(0); - sink.i64_const(ca.ca_alloc_fn_ptr); - sink.i64_store(mem64(CALL_FUNC_OFS)); - sink.local_get(0); - sink.i64_const(2); - sink.i64_store(mem64(CALL_NARGS_OFS)); - sink.local_get(0); - sink.i64_const(ca.callee_frame_bytes as i64); - sink.i64_store(mem64(CALL_ARGS_OFS)); - sink.local_get(0); - sink.i64_const(ca.callee_gcmap_ptr); - sink.i64_store(mem64(CALL_ARGS_OFS + SLOT_SIZE)); - sink.local_get(0); - sink.call(jit_call); - sink.local_get(0); - sink.i64_load(mem64(CALL_RESULT_OFS)); + if let Some(base) = residual_type_base { + sink.i64_const(ca.callee_frame_bytes as i64); + sink.i64_const(ca.callee_gcmap_ptr); + sink.i32_const(ca.ca_alloc_fn_ptr as i32); + sink.call_indirect(0, base + 2); + } else { + let jit_call = + jit_call_idx.expect("CA arm needs jit_call for the frame trampolines"); + sink.local_get(0); + sink.i64_const(ca.ca_alloc_fn_ptr); + sink.i64_store(mem64(CALL_FUNC_OFS)); + sink.local_get(0); + sink.i64_const(2); + sink.i64_store(mem64(CALL_NARGS_OFS)); + sink.local_get(0); + sink.i64_const(ca.callee_frame_bytes as i64); + sink.i64_store(mem64(CALL_ARGS_OFS)); + sink.local_get(0); + sink.i64_const(ca.callee_gcmap_ptr); + sink.i64_store(mem64(CALL_ARGS_OFS + SLOT_SIZE)); + sink.local_get(0); + sink.call(jit_call); + sink.local_get(0); + sink.i64_load(mem64(CALL_RESULT_OFS)); + } sink.i32_wrap_i64(); sink.i32_const(majit_backend::jitframe::FIRST_ITEM_OFFSET as i32); sink.i32_add(); @@ -2156,19 +2365,31 @@ fn build_function( sink.drop(); } // Pop the callee frame off the jitframe shadow stack (strict - // LIFO) via the jit_call trampoline (`wasm_jit_ca_pop_frame`). - sink.local_get(0); - sink.i64_const(ca.ca_pop_fn_ptr); - sink.i64_store(mem64(CALL_FUNC_OFS)); - sink.local_get(0); - sink.i64_const(1); - sink.i64_store(mem64(CALL_NARGS_OFS)); - sink.local_get(0); - sink.local_get(ca_cfp_local); - sink.i64_extend_i32_u(); - sink.i64_store(mem64(CALL_ARGS_OFS)); - sink.local_get(0); - sink.call(jit_call); + // LIFO) via `wasm_jit_ca_pop_frame` — same direct-vs-trampoline + // split as the alloc above (the pop only shrinks the shadow + // stack; it never allocates or collects). + if let Some(base) = residual_type_base { + sink.local_get(ca_cfp_local); + sink.i64_extend_i32_u(); + sink.i32_const(ca.ca_pop_fn_ptr as i32); + sink.call_indirect(0, base + 1); + sink.drop(); // returns 0; ignored + } else { + let jit_call = + jit_call_idx.expect("CA arm needs jit_call for the frame trampolines"); + sink.local_get(0); + sink.i64_const(ca.ca_pop_fn_ptr); + sink.i64_store(mem64(CALL_FUNC_OFS)); + sink.local_get(0); + sink.i64_const(1); + sink.i64_store(mem64(CALL_NARGS_OFS)); + sink.local_get(0); + sink.local_get(ca_cfp_local); + sink.i64_extend_i32_u(); + sink.i64_store(mem64(CALL_ARGS_OFS)); + sink.local_get(0); + sink.call(jit_call); + } // The callee recursion minor-collected; this bridge's other live // Ref locals are now stale. Reload them from the forwarded homes. // Skip the result `vi`: its local holds the just-read callee output @@ -2231,6 +2452,20 @@ fn build_function( } // store-on-def (end of loop) homes a Ref result, so the // direct path must NOT `continue` past it. + } else if let (Some(base), Some(nargs)) = + (residual_type_base, residual_call_void_word_arity(op)) + { + // Direct in-module word-ABI void residual call: the callee + // really is `(i64×n)->i64` (descr result_size == 8), so use + // the i64 family and drop the dummy result. + let call_args = &op.getarglist()[1..]; + for arg in call_args { + emit_resolve(&mut sink, constants, arg.to_opref()); + } + emit_resolve(&mut sink, constants, op.arg(0).to_opref()); + sink.i32_wrap_i64(); + sink.call_indirect(0, base + nargs as u32); + sink.drop(); } else { let jit_call = jit_call_idx.expect("CALL op present but jit_call not imported"); @@ -2290,7 +2525,6 @@ fn build_function( // `jit_call` trampoline to the `wasm_jit_alloc` helper, then write // the vtable / length fields with pointer-width (i32) stores. OpCode::New | OpCode::NewWithVtable => { - let jit_call = jit_call_idx.expect("New op present but jit_call not imported"); let vi = op.pos.get().raw(); // llmodel.py:778-782: size, type_id, vtable from the size descr. let descr = op.getdescr(); @@ -2299,32 +2533,131 @@ fn build_function( (sd.size() as i64, sd.type_id() as i64, sd.vtable()) }); - // func_ptr = wasm_jit_alloc - sink.local_get(0); - sink.i64_const(alloc_fn_ptr); - sink.i64_store(mem64(CALL_FUNC_OFS)); - // num_args = 2 - sink.local_get(0); - sink.i64_const(2); - sink.i64_store(mem64(CALL_NARGS_OFS)); - // arg0 = type_id - sink.local_get(0); - sink.i64_const(type_id); - sink.i64_store(mem64(CALL_ARGS_OFS)); - // arg1 = size - sink.local_get(0); - sink.i64_const(size); - sink.i64_store(mem64(CALL_ARGS_OFS + SLOT_SIZE)); - // call trampoline - sink.local_get(0); - sink.call(jit_call); - - if !OpRef::raw_is_constant(vi) { - // result pointer + // Inline nursery bump (rewrite.py malloc fast path, x86 + // `malloc_cond`): total = align8(max(header+size, MIN)); if + // `free + total` fits below `nursery_top`, commit the bump and + // write the header word (tid, no flags — young objects carry + // none) inline; otherwise fall to the collecting helper. + // Restricted to plain types (no destructor/weakref side-list) + // under the large-object threshold, exactly the helper's own + // fast path. + let total_size = { + use majit_gc::header::GcHeader; + ((GcHeader::SIZE + size as usize).max(GcHeader::MIN_NURSERY_OBJ_SIZE) + 7) & !7 + }; + let inline_nursery = nursery.filter(|na| { + total_size <= na.large_threshold + && u32::try_from(type_id).is_ok_and(|t| na.plain_tids.contains(&t)) + }); + if let (Some(base), Some(na)) = (residual_type_base, inline_nursery) { + // free = *nursery_free; new_free = free + total + sink.i32_const(na.free_addr as i32); + sink.i32_load(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + sink.local_tee(alloc_scratch_local); + sink.i32_const(total_size as i32); + sink.i32_add(); + // new_free > *nursery_top → slow path + sink.i32_const(na.top_addr as i32); + sink.i32_load(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + sink.i32_gt_u(); + sink.if_(BlockType::Result(ValType::I64)); + // Slow: collecting helper. The collection may have moved + // every other live Ref; reload them from their (forwarded) + // homes — only here, the fast path moves nothing. Skip the + // fresh result (still on the operand stack; its home is + // written by store-on-def below). + sink.i64_const(type_id); + sink.i64_const(size); + sink.i32_const(alloc_fn_ptr as i32); + sink.call_indirect(0, base + 2); + emit_reload_refs_from_homes( + &mut sink, + ref_homes, + (!OpRef::raw_is_constant(vi)).then_some(vi), + ); + sink.else_(); + // Commit: *nursery_free = free + total. + sink.i32_const(na.free_addr as i32); + sink.local_get(alloc_scratch_local); + sink.i32_const(total_size as i32); + sink.i32_add(); + sink.i32_store(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + // Header word: `GcHeader::new(tid)` — flags 0. + sink.local_get(alloc_scratch_local); + sink.i64_const(type_id); + sink.i64_store(MemArg { + offset: 0, + align: 3, + memory_index: 0, + }); + // Result payload pointer = free + header size. + sink.local_get(alloc_scratch_local); + sink.i32_const(majit_gc::header::GcHeader::SIZE as i32); + sink.i32_add(); + sink.i64_extend_i32_u(); + sink.end(); + if !OpRef::raw_is_constant(vi) { + sink.local_set(1 + vi); + } else { + sink.drop(); + } + } else if let Some(base) = residual_type_base { + // Direct in-module allocation: `wasm_jit_alloc(type_id, size)` + // is a plain `(i64,i64)->i64` table entry, so call it like an + // eligible residual call — no host hop. Its fn ptr is a table + // index on wasm32. + sink.i64_const(type_id); + sink.i64_const(size); + sink.i32_const(alloc_fn_ptr as i32); + sink.call_indirect(0, base + 2); + if !OpRef::raw_is_constant(vi) { + sink.local_set(1 + vi); + } else { + sink.drop(); + } + } else { + let jit_call = jit_call_idx.expect("New op present but jit_call not imported"); + // func_ptr = wasm_jit_alloc sink.local_get(0); - sink.i64_load(mem64(CALL_RESULT_OFS)); - sink.local_set(1 + vi); + sink.i64_const(alloc_fn_ptr); + sink.i64_store(mem64(CALL_FUNC_OFS)); + // num_args = 2 + sink.local_get(0); + sink.i64_const(2); + sink.i64_store(mem64(CALL_NARGS_OFS)); + // arg0 = type_id + sink.local_get(0); + sink.i64_const(type_id); + sink.i64_store(mem64(CALL_ARGS_OFS)); + // arg1 = size + sink.local_get(0); + sink.i64_const(size); + sink.i64_store(mem64(CALL_ARGS_OFS + SLOT_SIZE)); + // call trampoline + sink.local_get(0); + sink.call(jit_call); + + if !OpRef::raw_is_constant(vi) { + // result pointer + sink.local_get(0); + sink.i64_load(mem64(CALL_RESULT_OFS)); + sink.local_set(1 + vi); + } + } + if !OpRef::raw_is_constant(vi) { // llmodel.py:779-781 write_int_at_mem(res, vtable_offset, // WORD, vtable). The `ob_type` field is pointer-width: 4 // bytes on wasm32 (GuardClass reads it as i32), so store @@ -2347,12 +2680,15 @@ fn build_function( // The collecting allocation may have moved every other live // Ref; reload them from their (forwarded) homes. Skip the fresh // result — it was allocated after the collection and its home is - // written by store-on-def below. - let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); - emit_reload_refs_from_homes(&mut sink, ref_homes, skip); + // written by store-on-def below. The inline-bump path already + // emitted this reload inside its slow arm (the fast bump moves + // nothing). + if residual_type_base.is_none() || inline_nursery.is_none() { + let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); + emit_reload_refs_from_homes(&mut sink, ref_homes, skip); + } } OpCode::NewArray | OpCode::NewArrayClear => { - let jit_call = jit_call_idx.expect("NewArray op present but jit_call not imported"); let vi = op.pos.get().raw(); let descr = op.getdescr(); let ad = descr.as_ref().and_then(|d| d.as_array_descr()); @@ -2364,46 +2700,166 @@ fn build_function( .map_or(0i64, |ld| ld.offset() as i64); let type_id = ad.map_or(0i64, |ad| ad.type_id() as i64); - // func_ptr = wasm_jit_alloc_array - sink.local_get(0); - sink.i64_const(alloc_array_fn_ptr); - sink.i64_store(mem64(CALL_FUNC_OFS)); - // num_args = 5 - sink.local_get(0); - sink.i64_const(5); - sink.i64_store(mem64(CALL_NARGS_OFS)); - // arg0 = type_id - sink.local_get(0); - sink.i64_const(type_id); - sink.i64_store(mem64(CALL_ARGS_OFS)); - // arg1 = base_size - sink.local_get(0); - sink.i64_const(base_size); - sink.i64_store(mem64(CALL_ARGS_OFS + SLOT_SIZE)); - // arg2 = item_size - sink.local_get(0); - sink.i64_const(item_size); - sink.i64_store(mem64(CALL_ARGS_OFS + 2 * SLOT_SIZE)); - // arg3 = length (op.arg(0)) - sink.local_get(0); - emit_resolve(&mut sink, constants, op.arg(0).to_opref()); - sink.i64_store(mem64(CALL_ARGS_OFS + 3 * SLOT_SIZE)); - // arg4 = len_offset - sink.local_get(0); - sink.i64_const(len_offset); - sink.i64_store(mem64(CALL_ARGS_OFS + 4 * SLOT_SIZE)); - // call trampoline - sink.local_get(0); - sink.call(jit_call); - - if !OpRef::raw_is_constant(vi) { + // Inline nursery bump for a CONSTANT-length array of a plain + // type under the large-object threshold (same fast path as the + // `New` arm — the total is a compile-time constant, and the + // nursery is bulk-zeroed on reset so `NewArrayClear`'s cleared + // items come for free, exactly like the helper). Writes the + // header word and the length field inline. A runtime-length + // array keeps the helper call. + let inline_nursery_total = const_operand_value(constants, op.arg(0).to_opref()) + .and_then(|len| { + use majit_gc::header::GcHeader; + let len = usize::try_from(len).ok()?; + let payload = (base_size as usize) + .checked_add((item_size as usize).checked_mul(len)?)?; + let total = + ((GcHeader::SIZE + payload).max(GcHeader::MIN_NURSERY_OBJ_SIZE) + 7) + & !7; + let na = nursery.filter(|na| { + total <= na.large_threshold + && u32::try_from(type_id).is_ok_and(|t| na.plain_tids.contains(&t)) + })?; + Some((total, len, na)) + }); + if let (Some(base), Some((total_size, length, na))) = + (residual_type_base, inline_nursery_total) + { + // free = *nursery_free; new_free = free + total + sink.i32_const(na.free_addr as i32); + sink.i32_load(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + sink.local_tee(alloc_scratch_local); + sink.i32_const(total_size as i32); + sink.i32_add(); + sink.i32_const(na.top_addr as i32); + sink.i32_load(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + sink.i32_gt_u(); + sink.if_(BlockType::Result(ValType::I64)); + // Slow: collecting helper; reload the other live Refs from + // their (forwarded) homes — only here, the fast bump moves + // nothing. + sink.i64_const(type_id); + sink.i64_const(base_size); + sink.i64_const(item_size); + sink.i64_const(length as i64); + sink.i64_const(len_offset); + sink.i32_const(alloc_array_fn_ptr as i32); + sink.call_indirect(0, base + 5); + emit_reload_refs_from_homes( + &mut sink, + ref_homes, + (!OpRef::raw_is_constant(vi)).then_some(vi), + ); + sink.else_(); + // Commit: *nursery_free = free + total. + sink.i32_const(na.free_addr as i32); + sink.local_get(alloc_scratch_local); + sink.i32_const(total_size as i32); + sink.i32_add(); + sink.i32_store(MemArg { + offset: 0, + align: 2, + memory_index: 0, + }); + // Header word: `GcHeader::new(tid)` — flags 0. + sink.local_get(alloc_scratch_local); + sink.i64_const(type_id); + sink.i64_store(MemArg { + offset: 0, + align: 3, + memory_index: 0, + }); + // Length field (usize, 4 bytes on wasm32) at + // `payload + len_offset`. + sink.local_get(alloc_scratch_local); + sink.i32_const(length as i32); + sink.i32_store(MemArg { + offset: majit_gc::header::GcHeader::SIZE as u64 + len_offset as u64, + align: 2, + memory_index: 0, + }); + // Result payload pointer = free + header size. + sink.local_get(alloc_scratch_local); + sink.i32_const(majit_gc::header::GcHeader::SIZE as i32); + sink.i32_add(); + sink.i64_extend_i32_u(); + sink.end(); + if !OpRef::raw_is_constant(vi) { + sink.local_set(1 + vi); + } else { + sink.drop(); + } + } else if let Some(base) = residual_type_base { + // Direct in-module allocation, like the `New` arm: + // `wasm_jit_alloc_array(type_id, base_size, item_size, + // length, len_offset)` is a `(i64×5)->i64` table entry. + sink.i64_const(type_id); + sink.i64_const(base_size); + sink.i64_const(item_size); + emit_resolve(&mut sink, constants, op.arg(0).to_opref()); + sink.i64_const(len_offset); + sink.i32_const(alloc_array_fn_ptr as i32); + sink.call_indirect(0, base + 5); + if !OpRef::raw_is_constant(vi) { + sink.local_set(1 + vi); + } else { + sink.drop(); + } + } else { + let jit_call = + jit_call_idx.expect("NewArray op present but jit_call not imported"); + // func_ptr = wasm_jit_alloc_array sink.local_get(0); - sink.i64_load(mem64(CALL_RESULT_OFS)); - sink.local_set(1 + vi); + sink.i64_const(alloc_array_fn_ptr); + sink.i64_store(mem64(CALL_FUNC_OFS)); + // num_args = 5 + sink.local_get(0); + sink.i64_const(5); + sink.i64_store(mem64(CALL_NARGS_OFS)); + // arg0 = type_id + sink.local_get(0); + sink.i64_const(type_id); + sink.i64_store(mem64(CALL_ARGS_OFS)); + // arg1 = base_size + sink.local_get(0); + sink.i64_const(base_size); + sink.i64_store(mem64(CALL_ARGS_OFS + SLOT_SIZE)); + // arg2 = item_size + sink.local_get(0); + sink.i64_const(item_size); + sink.i64_store(mem64(CALL_ARGS_OFS + 2 * SLOT_SIZE)); + // arg3 = length (op.arg(0)) + sink.local_get(0); + emit_resolve(&mut sink, constants, op.arg(0).to_opref()); + sink.i64_store(mem64(CALL_ARGS_OFS + 3 * SLOT_SIZE)); + // arg4 = len_offset + sink.local_get(0); + sink.i64_const(len_offset); + sink.i64_store(mem64(CALL_ARGS_OFS + 4 * SLOT_SIZE)); + // call trampoline + sink.local_get(0); + sink.call(jit_call); + + if !OpRef::raw_is_constant(vi) { + sink.local_get(0); + sink.i64_load(mem64(CALL_RESULT_OFS)); + sink.local_set(1 + vi); + } + } + // `wasm_jit_alloc_array` collects; reload other live Refs. The + // inline-bump path already emitted this inside its slow arm. + if residual_type_base.is_none() || inline_nursery_total.is_none() { + let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); + emit_reload_refs_from_homes(&mut sink, ref_homes, skip); } - // `wasm_jit_alloc_array` collects; reload other live Refs. - let skip = (!OpRef::raw_is_constant(vi)).then_some(vi); - emit_reload_refs_from_homes(&mut sink, ref_homes, skip); } // ── Misc ── @@ -2507,11 +2963,11 @@ fn build_function( if has_loop { sink.end(); // end loop sink.end(); // end block - } else if ca_straightline_dispatch { - sink.end(); // end exit block A (no `loop` in a Label-less CA loop) + } else if straightline_dispatch { + sink.end(); // end exit block A (Label-less dispatch trace, no `loop`) } - // Epilogue bridge dispatch (loop traces only). Control reaches here only + // Epilogue bridge dispatch. Control reaches here only // after a guard `br`'d out of the exit block, having written its // `fail_index` into `frame[0]`. Look up that guard's bridge slot in the // shared cell array; if a bridge has been compiled (slot != 0), tail into @@ -2521,10 +2977,9 @@ fn build_function( // (no bridge yet) this is inert and behavior is unchanged. if bridge_dispatch { // slot = *(cells_base + (fail_index - fail_index_base) * 4) - // The cell array is local to this trace (one i32 per local guard), so a - // bridge whose `frame[0]` carries a base-offset global index subtracts - // the base back to a local cell index. Loops pass base 0 → no subtract, - // keeping their module byte-identical. + // The cell array is local to this trace (one i32 per local guard); + // `frame[0]` carries the GLOBAL fail index, so subtract this trace's + // base back to a local cell index. sink.i32_const(cells_base as i32); sink.local_get(0); sink.i64_load(mem64(0)); // frame[0] = fail_index @@ -2555,15 +3010,14 @@ fn build_function( /// A peeled loop — real work (the unrolled first iteration = preamble) precedes /// the loop-header LABEL — whether it carries one LABEL or several. `loop` is -/// emitted at the LAST label, so `build_function` wraps the preamble in the -/// resume-at-LABEL dispatch (keyed on the frame dispatch-key slot) and a -/// loop-closing bridge re-enters AT that last label, skipping the preamble, -/// in-module. `build_function` keys its preamble-skip wrapper on this predicate; -/// `compile_loop` records it on `CompiledWasmLoop`. The decline `compile_bridge` -/// lifts is finer-grained: a single-label source is accepted directly -/// (`is_single_label_peeled`), a multi-label source only when the bridge's JUMP -/// targets that last label (recovered from its descr) — every other multi-label -/// target needs the deferred br_table and stays declined. +/// emitted at the LAST label, so `build_function` wraps the trace in the +/// resume-at-LABEL entry `br_table` (keyed on the frame dispatch-key slot, +/// key = label ordinal + 1) and a loop-closing bridge re-enters at ANY of the +/// loop's labels, in-module. `build_function` keys its wrapper on this +/// predicate; `compile_loop` records it on `CompiledWasmLoop` as +/// `has_preamble`. `compile_bridge` accepts a loop-closing bridge only when +/// its JUMP's descr identifies one of the source loop's OWN labels +/// (`label_descrs`) with matching arity and a resume-safe live set. pub fn is_resumable_peeled(ops: &[Op]) -> bool { let Some(last_label) = ops.iter().rposition(|op| op.opcode == OpCode::Label) else { return false; @@ -2573,16 +3027,69 @@ pub fn is_resumable_peeled(ops: &[Op]) -> bool { .any(|op| op.opcode != OpCode::Label) } -/// The single-label subset of `is_resumable_peeled`: exactly one LABEL. A -/// loop-closing bridge into such a loop always targets that sole label, so -/// `compile_bridge` accepts it without recovering the target ordinal. Kept a -/// distinct predicate (not folded into `is_resumable_peeled`) so the bridge -/// accept-condition can treat single- and multi-label sources differently. +/// The single-label subset of `is_resumable_peeled`: exactly one LABEL. +/// No longer consulted by the bridge accept-condition (which resolves the +/// JUMP's target label by descr identity uniformly); kept as a shape +/// predicate for tests. pub fn is_single_label_peeled(ops: &[Op]) -> bool { let label_count = ops.iter().filter(|op| op.opcode == OpCode::Label).count(); is_resumable_peeled(ops) && label_count == 1 } +/// Argument count of each `LABEL`, in ordinal order (the same ordinals +/// `compile_loop` stamps via `set_label_block_id`). `compile_bridge` declines +/// a loop-closing bridge whose JUMP arity differs from its target label's +/// count, since the resume loader reads exactly that many positional frame +/// slots. +pub fn label_arg_counts(ops: &[Op]) -> Vec { + ops.iter() + .filter(|op| op.opcode == OpCode::Label) + .map(|op| op.getarglist().len()) + .collect() +} + +/// Per-label resume safety, in ordinal order: label `j` is safe to resume at +/// when every op after it references only values that are constants, defined +/// after the label, or listed in the label's own args — i.e. the label's args +/// are the complete live set, so the resume loader reconstructs every value +/// the remainder of the trace reads. A value defined before the label and +/// read after it without being a label arg would resume as a null local (the +/// resume path skips the entry loader and every earlier segment). Guard fail +/// args count as reads — they spill into the deopt frame. +pub fn label_resume_safety(ops: &[Op]) -> Vec { + ops.iter() + .enumerate() + .filter(|(_, op)| op.opcode == OpCode::Label) + .map(|(p, label)| { + let mut live: std::collections::HashSet = label + .getarglist() + .iter() + .map(|a| a.to_opref()) + .filter(|r| *r != OpRef::NONE && !r.is_constant()) + .map(|r| r.raw()) + .collect(); + for op in &ops[p + 1..] { + let args = op.getarglist(); + let arg_reads = args.iter().map(|a| a.to_opref()); + let fail_reads = op + .getfailargs() + .map(|fa| fa.iter().map(|a| a.to_opref()).collect::>()) + .unwrap_or_default(); + for r in arg_reads.chain(fail_reads) { + if r != OpRef::NONE && !r.is_constant() && !live.contains(&r.raw()) { + return false; + } + } + let res = op.pos.get(); + if res != OpRef::NONE && !res.is_constant() { + live.insert(res.raw()); + } + } + true + }) + .collect() +} + fn find_label_args(ops: &[Op]) -> Vec { // The JUMP branches back to the loop-header label, which is the LAST // label in a peeled trace (an outer entry label may precede it). The @@ -2611,6 +3118,16 @@ fn emit_resolve( } } +/// Compile-time value of a constant operand (what `emit_resolve` would push +/// as `i64.const`), or `None` for a runtime value. +fn const_operand_value(constants: &majit_ir::VecMap, opref: OpRef) -> Option { + opref.is_constant().then(|| { + opref + .inline_const_bits() + .unwrap_or_else(|| constants.get(&opref.raw()).copied().unwrap_or(0)) + }) +} + /// Extract field offset from op's descr (FieldDescr). fn field_offset_from_descr(op: &Op) -> u64 { let __descr_arc_descr = op.getdescr(); diff --git a/majit/majit-backend-wasm/src/failguard.rs b/majit/majit-backend-wasm/src/failguard.rs index 0ff260ed5dd..d935bac03a4 100644 --- a/majit/majit-backend-wasm/src/failguard.rs +++ b/majit/majit-backend-wasm/src/failguard.rs @@ -64,17 +64,165 @@ pub struct WasmFrameData { pub exc_value: i64, } +/// A resumable `LABEL` of a compiled loop, published in `LABEL_TARGETS` so a +/// loop-closing bridge can chain into ANY compiled loop's label in-module +/// (jump-to-existing-trace), not only its own source loop's. Keyed by the +/// label's loop-target descr identity (`Arc::as_ptr`), which the JUMP shares. +#[derive(Clone, Copy, Debug)] +pub struct LabelTarget { + /// Table slot of the owning loop's compiled function. + pub func_handle: u32, + /// Resume dispatch key (`label ordinal + 1`) the bridge's JUMP writes. + pub key: u32, + /// The label's arg count — the resume loader reads exactly this many + /// positional frame slots, so the JUMP arity must equal it. + pub num_args: usize, + /// Whether the label's args are the complete live set of the owning + /// trace's remainder (`codegen::label_resume_safety`). + pub resume_safe: bool, + /// Whether this is the owning loop's LAST label (the loop header). A + /// bridge landing here re-runs no segment code before the `loop`, so the + /// livelock advance-check applies; earlier labels execute the peeled + /// segment, which advances the state by itself. + pub is_last_label: bool, + /// Ref-home slot count of the owning loop, for the chain-soundness check: + /// a chained trace runs in the frame `execute_token` sized for the loop + /// the chain ENTERED through, so a tail-call may only target a loop whose + /// Ref-home region fits `max(source loop's homes, FRAME_REF_HOME_FLOOR)` + /// — inductively, every loop in a chain then fits the entry frame (which + /// is sized to at least the floor). Ref homes are the ONLY variable frame + /// requirement: value slots are bounded by `codegen`'s + /// `CALL_AREA_FIRST_SLOT` decline, below the constant + /// `MIN_FRAME_BYTES / 8` value region every host frame carries. + pub num_ref_homes: usize, +} + +/// Minimum Ref-home slot count `execute_token` sizes every host frame for +/// while bridge chaining is enabled (see `LabelTarget::num_ref_homes`). +/// Chains between traces whose home counts stay at or under this floor need +/// no source-vs-target comparison at all, which is what lifts most frame-fit +/// bridge declines. Costs `8` bytes + one GC-root registration per slot per +/// `execute_token` call. +pub const FRAME_REF_HOME_FLOOR: usize = 64; + +/// Global `frame[0]` fail-index space. +/// +/// Cross-trace chaining (`LABEL_TARGETS`) means the module that last wrote +/// `frame[0]` is not necessarily the loop `execute_token` entered: a bridge's +/// terminal JUMP may tail-call a SIBLING loop, whose guards then write THEIR +/// exit indices. Per-loop index spaces would make those writes ambiguous at +/// the host — resolving `frame[0]` against the entry loop's own `fail_descrs` +/// picks a wrong descr (wrong arg types/resume ⇒ type confusion). So every +/// compile (`compile_loop` and `compile_bridge`) allocates its exits from this +/// one global space: it passes the registry length as codegen's +/// `fail_index_base`, guards write `base + local` into `frame[0]`, and the +/// registered descrs land at exactly those registry positions — any `frame[0]` +/// then resolves here regardless of which chained module wrote it. The +/// per-guard bridge-cell epilogue keeps its local indexing by subtracting the +/// owning module's base (`codegen`'s cell lookup). +/// +/// Entries are never removed: a dropped loop's modules are unreachable (its +/// label targets are retracted and its token is gone), so its entries are just +/// retained memory, bounded by the total number of compiled exits. +static FAIL_DESCR_REGISTRY: std::sync::Mutex>>> = + std::sync::Mutex::new(None); + +/// The next free global fail index — pass as `fail_index_base` to +/// `codegen::build_wasm_module`, then register the built descrs with +/// `register_fail_descrs`. The wasm host is single-threaded, so no other +/// compile can interleave between the two calls. +pub fn fail_descr_base() -> u32 { + FAIL_DESCR_REGISTRY + .lock() + .unwrap() + .as_ref() + .map_or(0, |v| v.len() as u32) +} + +/// Append a compile's exit descrs to the global space. Each descr's +/// `fail_index` (already base-offset by `build_wasm_module`) must equal the +/// registry position it lands at. +pub fn register_fail_descrs(descrs: &[Arc]) { + let mut reg = FAIL_DESCR_REGISTRY.lock().unwrap(); + let vec = reg.get_or_insert_with(Default::default); + for d in descrs { + debug_assert_eq!( + d.fail_index as usize, + vec.len(), + "fail descr registered out of lockstep with its global fail_index" + ); + vec.push(Arc::clone(d)); + } +} + +/// Resolve a `frame[0]` value through the global fail-index space. +pub fn global_fail_descr(fail_index: u32) -> Option> { + FAIL_DESCR_REGISTRY + .lock() + .unwrap() + .as_ref() + .and_then(|v| v.get(fail_index as usize).cloned()) +} + +/// Global `label descr identity → LabelTarget` registry (see `LabelTarget`). +/// The wasm host is single-threaded; the `Mutex` is for `static` soundness +/// only. `compile_loop` inserts every resumable label of a peeled loop; +/// `CompiledWasmLoop::drop` removes its own entries (guarded by +/// `func_handle`, so a recompile that re-stamped the same descr keeps the +/// replacement's entry). +pub static LABEL_TARGETS: std::sync::Mutex>> = + std::sync::Mutex::new(None); + +/// Look up a label target by descr identity. +pub fn label_target(descr_id: usize) -> Option { + LABEL_TARGETS + .lock() + .unwrap() + .as_ref() + .and_then(|m| m.get(&descr_id).copied()) +} + +/// Publish a label target (see `LABEL_TARGETS`). +pub fn publish_label_target(descr_id: usize, target: LabelTarget) { + LABEL_TARGETS + .lock() + .unwrap() + .get_or_insert_with(Default::default) + .insert(descr_id, target); +} + +/// Guard-dispatch metadata of a bridge chained onto a loop, kept on the +/// source loop's `CompiledWasmLoop.chained_trace_meta` keyed by the bridge's +/// backend `trace_id`. Lets `compile_bridge` chain a NESTED sub-bridge onto a +/// guard that lives inside an already-chained bridge: the failing guard's +/// meta descr carries `(trace_id, per-trace fail_index)`, and this record +/// supplies the owning bridge's cell array and livelock advance flags — the +/// same data `CompiledWasmLoop` holds for the loop's own guards. +pub struct ChainedTraceMeta { + /// Base address of the bridge's per-guard bridge-slot cell array + /// (`CompiledWasmLoop::bridge_cells_base` analog); `0` = no dispatch. + pub cells_base: u32, + /// Cell count = the bridge's own guard count. + pub num_cells: usize, + /// Per-guard, per-fail-arg induction-advance flags + /// (`CompiledWasmLoop::guard_fail_arg_advanced` analog). + pub guard_fail_arg_advanced: Vec>, +} + /// Compiled wasm loop metadata, stored in `JitCellToken.compiled`. pub struct CompiledWasmLoop { pub trace_id: u64, pub input_types: Vec, pub func_handle: u32, - /// Guard/finish exit descriptors, indexed by the `fail_index` written into - /// `frame[0]`. `compile_bridge` appends its bridge's descrs here (past the - /// loop's own `[0, num_guards)` range) so `execute_token` resolves loop and - /// chained-bridge exits through one array. `RefCell` because the append - /// happens through the shared `&JitCellToken` the bridge attaches to; the - /// wasm host is single-threaded so no cross-thread access occurs. + /// This loop's own guard/finish exit descriptors (positions `[0, + /// num_guard_cells)`, per-trace order), followed by the descr slices of + /// every chained bridge `compile_bridge` appended (positional bookkeeping + /// for `bridge_descr_ranges` — layouts and jitcounter hashes). `frame[0]` + /// exit resolution does NOT index this vec: exit indices live in the + /// GLOBAL fail-index space (`register_fail_descrs`), because a cross-trace + /// chain can exit through a sibling loop's guard. `RefCell` because the + /// append happens through the shared `&JitCellToken` the bridge attaches + /// to; the wasm host is single-threaded so no cross-thread access occurs. pub fail_descrs: RefCell>>, pub num_inputs: usize, pub max_output_slots: usize, @@ -96,37 +244,42 @@ pub struct CompiledWasmLoop { /// True when this is a peeled loop (`codegen::is_resumable_peeled`) — there /// is real work (a preamble = the unrolled first iteration) before the last /// `LABEL`, single- or multi-label. Such a loop carries the resume-at-LABEL - /// preamble-skip dispatch and resumes at the LAST label (where the `loop` - /// is). A loop-closing bridge re-enters through the loop's table slot (the - /// function entry); for a peeled loop, re-running the preamble against - /// mid-loop state would never advance the induction variable — an infinite - /// loop. `compile_bridge` therefore declines a loop-closing bridge into a - /// peeled loop UNLESS it resumes at that last label: always so for a - /// single-label source (`is_single_label_peeled`), and for a multi-label - /// source only when the bridge's JUMP targets `last_label_block_id`. + /// entry `br_table` (key = label ordinal + 1) so a loop-closing bridge can + /// re-enter at any of its labels. A loop-closing bridge re-enters through + /// the loop's table slot (the function entry); for a peeled loop, + /// re-running the preamble against mid-loop state would never advance the + /// induction variable — an infinite loop. `compile_bridge` therefore + /// declines a loop-closing bridge UNLESS its JUMP's + /// target label resolves to a published, resumable `LabelTarget`. pub has_preamble: bool, - /// True when this peeled loop has exactly one `LABEL` - /// (`codegen::is_single_label_peeled`). A loop-closing bridge into such a - /// loop always targets that sole label, so `compile_bridge` accepts it - /// without recovering the JUMP's target ordinal. - pub is_single_label_peeled: bool, - /// `label_block_id` of the LAST `LABEL` (= label_count − 1), the one carrying - /// the wasm `loop`. A loop-closing bridge into a multi-label source is - /// accepted only when its JUMP descr's recovered `label_block_id` equals this - /// — i.e. the bridge resumes at the label the resume dispatch lands on. - pub last_label_block_id: u32, - /// Argument count of the LAST `LABEL`. The accept-condition declines a bridge - /// whose closing JUMP arity differs from this, since the resume loader reads - /// exactly this many positional frame slots (an arity mismatch would resume - /// with stale/missing induction values). - pub last_label_num_args: usize, - /// `(source_fail_index, start, count)` ranges into `fail_descrs` for each - /// chained bridge `compile_bridge` appended (lib.rs extend site). Lets - /// `compiled_bridge_fail_descr_layouts` / `store_bridge_guard_hashes` map a - /// source guard back to its bridge's appended descr slice — the wasm analog - /// of dynasm's `lookup_bridge_addr` (runner.rs). Recorded in lockstep with - /// the `extend`, inside the same `borrow_mut` critical section. - pub bridge_descr_ranges: RefCell>, + /// Descr identity (`Arc::as_ptr`) of each `LABEL`, in ordinal order; `0` + /// for a descr-less label. `compile_bridge` resolves a closing JUMP's + /// target label by matching its descr identity against this list — a JUMP + /// whose descr is not here targets ANOTHER trace's label (e.g. a sibling + /// retrace specialization, whose start label carries the same stamped + /// ordinal) and must not be chained into this loop. + pub label_descrs: Vec, + /// Per-guard (indexed by this loop's own `fail_index`), per-fail-arg: + /// whether the value was produced by induction-advancing arithmetic after + /// the loop-header label — fresh in the failing iteration. Consulted by + /// `compile_bridge`'s livelock check: a loop-closing bridge that JUMPs + /// such a fail arg verbatim still advances the chained cycle. + pub guard_fail_arg_advanced: Vec>, + /// `(source_trace_id, source_fail_index, start, count)` ranges into + /// `fail_descrs` for each chained bridge `compile_bridge` appended (lib.rs + /// extend site). Lets `compiled_bridge_fail_descr_layouts` / + /// `store_bridge_guard_hashes` map a source guard back to its bridge's + /// appended descr slice — the wasm analog of dynasm's + /// `lookup_bridge_addr` (runner.rs). Keyed by BOTH the source guard's + /// owning trace and its per-trace fail index: with nested chaining, the + /// loop's guard `k` and a chained bridge's guard `k` are distinct sources. + /// Recorded in lockstep with the `extend`, inside the same `borrow_mut` + /// critical section. + pub bridge_descr_ranges: RefCell>, + /// Guard-dispatch metadata of every bridge chained onto this loop, keyed + /// by the bridge's backend `trace_id` (see [`ChainedTraceMeta`]). Lets a + /// guard INSIDE a chained bridge chain its own nested sub-bridge. + pub chained_trace_meta: RefCell>, /// Owns this loop's per-guard bridge-slot cell array so it is freed on /// `Drop`; `bridge_cells_base` aliases its heap address (stable across the /// struct move). `None` when the trace has no in-module dispatch. @@ -144,4 +297,33 @@ pub struct CompiledWasmLoop { /// accepts a CA bridge; `Cell` because the source token is shared (`&`) and /// the wasm host is single-threaded. pub ca_bridge_ref_homes: Cell, + /// Set when `compile_bridge` accepts a self-recursive `CallAssemblerR` + /// bridge (`PYRE_WASM_CA`) for this loop. While set, `compile_bridge` + /// declines chaining any FURTHER bridge into this recursion (the guard + /// falls back to host round-trips): a chained bridge deopting inside the + /// CA recursion trips a resume seam that reads a clobbered class — see + /// the decline site for the failing suite shapes. + pub ca_active: Cell, +} + +impl Drop for CompiledWasmLoop { + fn drop(&mut self) { + // Retract this loop's published label targets so a later bridge + // cannot chain into a dropped loop's stale table slot. Guarded by + // `func_handle`: a recompile that re-stamped the same descr onto its + // replacement loop has already overwritten the entry, which must + // survive the old loop's drop. + let mut reg = LABEL_TARGETS.lock().unwrap(); + if let Some(map) = reg.as_mut() { + for &id in &self.label_descrs { + if id != 0 { + if let Some(t) = map.get(&id) { + if t.func_handle == self.func_handle { + map.remove(&id); + } + } + } + } + } + } } diff --git a/majit/majit-backend-wasm/src/lib.rs b/majit/majit-backend-wasm/src/lib.rs index 054287ba653..4df02ee0fd8 100644 --- a/majit/majit-backend-wasm/src/lib.rs +++ b/majit/majit-backend-wasm/src/lib.rs @@ -31,9 +31,9 @@ use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; /// did not resolve (target_ord None), 9 = target_ord Some but != last label, /// 10 = arity mismatch, 11 = loop-closing bridge advances no loop-carried value /// (guard side-trace that would livelock the chained loop). -pub static BRIDGE_DIAG: [AtomicU64; 14] = { +pub static BRIDGE_DIAG: [AtomicU64; 16] = { const Z: AtomicU64 = AtomicU64::new(0); - [Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z] + [Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z] }; /// Read a `BRIDGE_DIAG` tally (saturating index). Surfaced to the host through @@ -90,7 +90,43 @@ fn is_inductive_arith(opcode: majit_ir::OpCode) -> bool { ) } -use failguard::{CompiledWasmLoop, WasmFailDescr, WasmFrameData}; +/// Per-guard (per-trace order), per-fail-arg: whether the value was produced +/// by induction-advancing arithmetic in the part of the trace that re-runs on +/// every pass — the ops after the loop-header (last) LABEL, or the WHOLE trace +/// when it has no LABEL (a bridge, or a Label-less recursion loop, whose body +/// runs in full each pass). Such a fail arg is fresh in the failing iteration, +/// so a loop-closing bridge that JUMPs it verbatim still advances the chained +/// loop⇄bridge cycle (`compile_bridge`'s livelock check). +fn guard_fail_args_advanced( + ops: &[majit_ir::Op], + guard_exits: &[codegen::GuardExit], +) -> Vec> { + let start = ops + .iter() + .rposition(|op| op.opcode == majit_ir::OpCode::Label) + .map_or(0, |p| p + 1); + let advanced_ids: std::collections::HashSet = ops[start..] + .iter() + .filter(|op| is_inductive_arith(op.opcode)) + .map(|op| op.pos.get()) + .filter(|r| *r != majit_ir::OpRef::NONE && !r.is_constant()) + .map(|r| r.raw()) + .collect(); + guard_exits + .iter() + .map(|g| { + g.fail_arg_refs + .iter() + .map(|r| !r.is_constant() && advanced_ids.contains(&r.raw())) + .collect() + }) + .collect() +} + +use failguard::{ + ChainedTraceMeta, CompiledWasmLoop, LabelTarget, WasmFailDescr, WasmFrameData, fail_descr_base, + global_fail_descr, label_target, publish_label_target, register_fail_descrs, +}; use majit_backend::{AsmInfo, BackendError, DeadFrame, JitCellToken}; use majit_gc::GcAllocator; use majit_ir::{FailDescr, GcRef, InputArg, Op, OpRc, Value}; @@ -179,6 +215,59 @@ pub fn active_gc_heap_stats() -> (usize, usize) { with_wasm_active_gc(|gc| gc.heap_byte_stats()).unwrap_or((0, 0)) } +/// Diagnostic: `(minor_collections, major_collections)` of the active GC, or +/// `(0, 0)` when none is installed. Companion to [`active_gc_heap_stats`]. +pub fn active_gc_collection_counts() -> (usize, usize) { + with_wasm_active_gc(|gc| gc.collection_counts()).unwrap_or((0, 0)) +} + +/// Assemble the inline nursery-bump parameters for this trace's `New` / +/// `NewWithVtable` ops (rewrite.py malloc-fast-path eligibility over the +/// gc.py:525-531 nursery address surface), or `None` when no GC is active, +/// the `gc_stress` feature is compiled in (the fast path would bypass its +/// per-allocation stress collections), or no allocation op qualifies. +fn nursery_alloc_params(ops: &[Op]) -> Option { + if majit_gc::gc_stress_enabled() || !wasm_inline_alloc_enabled() { + return None; + } + let tids: std::collections::HashSet = ops + .iter() + .filter_map(|op| match op.opcode { + majit_ir::OpCode::New | majit_ir::OpCode::NewWithVtable => { + Some(op.getdescr()?.as_size_descr()?.type_id()) + } + majit_ir::OpCode::NewArray | majit_ir::OpCode::NewArrayClear => { + Some(op.getdescr()?.as_array_descr()?.type_id()) + } + _ => None, + }) + .collect(); + if tids.is_empty() { + return None; + } + with_wasm_active_gc(|gc| { + let free_addr = gc.nursery_free_addr(); + let top_addr = gc.nursery_top_addr(); + if free_addr == 0 || top_addr == 0 { + return None; + } + let plain_tids: std::collections::HashSet = tids + .iter() + .copied() + .filter(|&t| gc.type_alloc_is_plain(t)) + .collect(); + if plain_tids.is_empty() { + return None; + } + Some(codegen::NurseryAllocParams { + free_addr: free_addr as u32, + top_addr: top_addr as u32, + large_threshold: gc.max_nursery_object_size(), + plain_tids, + }) + })? +} + /// `majit_gc::CollectOldgenFn` installed by `set_gc_allocator`. Drives the /// interpreter-safepoint non-moving old-gen major (`gc_interp::safepoint`, /// default-on on wasm) through the wasm-thread-local GC. Needs mutable access, @@ -318,16 +407,36 @@ pub extern "C" fn wasm_jit_write_barrier(obj: i64) -> i64 { 0 } +thread_local! { + /// Live CA callee frames in recursion order (mirrors the CA entries on + /// the jf shadow stack): `(frame_addr, alloc_capacity_bytes)`. Popped + /// into [`CA_FRAME_POOL`] by `wasm_jit_ca_pop_frame`. + static CA_ACTIVE_FRAMES: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + /// LIFO pool of retired CA callee frames available for reuse. Strict CA + /// recursion order means the top entry is almost always the geometry the + /// next call wants, so the whole recursion runs on a handful of frames + /// instead of allocating one per call. Entries stay registered as libc + /// jitframes and are never freed (bounded by peak recursion depth). + static CA_FRAME_POOL: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; +} + /// Self-recursive CALL_ASSEMBLER (`PYRE_WASM_CA`) callee-frame allocation -/// trampoline. Allocates the callee's execution frame as a real GC-managed -/// `JitFrame` in **old-gen** (non-moving ⇒ the frame pointer the callee holds in -/// its wasm local 0 never dangles across a collection; `alloc_in_oldgen` never -/// itself collects, so the caller's still-unrooted Ref locals stay valid), -/// initializes its header + per-frame `jf_gcmap` (covering the callee's input + -/// home Ref slots), and pushes it on the jitframe shadow stack so a -/// mid-recursion collection forwards those Refs via the gcmap custom trace. +/// helper. Allocates the callee's execution frame as a **libc-jitframe** +/// (malloc memory, like dynasm's `execute_token` calloc frames — registered +/// via `register_libc_jitframe` so the collector's jf-root walk traces its +/// gcmap-marked Ref slots), initializes its header + per-frame `jf_gcmap` +/// (covering the callee's input + home Ref slots), and pushes it on the +/// jitframe shadow stack so a mid-recursion collection forwards those Refs +/// via the registered libc-jitframe tracer. Malloc (not nursery/old-gen) +/// keeps the frame non-moving AND off the GC's `bytes_made_old_since_cycle` +/// accounting — a per-call old-gen frame made every recursion level look +/// like heap growth and drove back-to-back major collections (fib: 1704 +/// majors for one bench run). Frames are pooled LIFO on pop and re-zeroed on +/// reuse, so steady-state recursion performs no allocator calls at all. /// Returns the frame base (codegen adds `FIRST_ITEM_OFFSET` for the -/// bespoke-layout frame pointer), or 0 if no GC / type id is installed. +/// bespoke-layout frame pointer), or 0 on allocation failure. /// /// Each callee frame self-describes through its own per-frame gcmap, so /// mixed-geometry frames from distinct CA bridges are each forwarded by their @@ -335,35 +444,60 @@ pub extern "C" fn wasm_jit_write_barrier(obj: i64) -> i64 { /// frame's interior as a smaller frame's slots. pub extern "C" fn wasm_jit_ca_alloc_frame(frame_bytes: i64, gcmap_ptr: i64) -> i64 { use majit_backend::jitframe::JitFrame; - let tid = wasm_jitframe_tid(); - if tid == 0 { - return 0; - } let depth = frame_bytes as usize / std::mem::size_of::(); - let frame = wasm_alloc_oldgen_typed(tid, JitFrame::alloc_size(depth)); - if frame.0 == 0 { - return 0; - } - let jf = frame.0 as *mut JitFrame; + let alloc_size = JitFrame::alloc_size(depth); + // Reuse the pool top when it is large enough (`>=` also covers a smaller + // bridge nesting inside a larger one). A too-small top is left in place — + // the fresh frame below re-pools on top of it in LIFO order. + let reused = CA_FRAME_POOL.with(|pool| { + let mut pool = pool.borrow_mut(); + match pool.last() { + Some(&(_, cap)) if cap >= alloc_size => pool.pop(), + _ => None, + } + }); + let (addr, cap) = match reused { + Some((addr, cap)) => { + // `JitFrame::init` expects zero-filled memory, and the gcmap-marked + // slots must not expose the previous run's stale Refs to a tracer. + unsafe { std::ptr::write_bytes(addr as *mut u8, 0, alloc_size) }; + (addr, cap) + } + None => { + let layout = std::alloc::Layout::from_size_align(alloc_size, 16) + .expect("CA frame layout overflow"); + let p = unsafe { std::alloc::alloc_zeroed(layout) }; + if p.is_null() { + return 0; + } + majit_gc::shadow_stack::register_libc_jitframe(p as usize); + (p as usize, alloc_size) + } + }; + let jf = addr as *mut JitFrame; unsafe { JitFrame::init(jf, std::ptr::null(), depth); (*jf).jf_gcmap = gcmap_ptr as *const u8; } - majit_gc::shadow_stack::push_jf(frame); - frame.0 as i64 + CA_ACTIVE_FRAMES.with(|v| v.borrow_mut().push((addr, cap))); + majit_gc::shadow_stack::push_jf(GcRef(addr)); + addr as i64 } /// Companion to [`wasm_jit_ca_alloc_frame`]: pop the top jitframe shadow-stack -/// entry on CA-arm exit (the callee frame just ran to finish/deopt). The CA -/// recursion is strict LIFO — each level pushes one frame before its -/// `call_indirect` and pops after, and a deopt resume runs on the host's own -/// shadow stack — so removing the top entry releases exactly this callee's -/// frame. +/// entry on CA-arm exit (the callee frame just ran to finish/deopt) and move +/// the frame into the reuse pool. The CA recursion is strict LIFO — each level +/// pushes one frame before its `call_indirect` and pops after, and a deopt +/// resume runs on the host's own shadow stack — so removing the top entry +/// releases exactly this callee's frame. pub extern "C" fn wasm_jit_ca_pop_frame(_frame_base: i64) -> i64 { let depth = majit_gc::shadow_stack::jf_depth(); if depth > 0 { majit_gc::shadow_stack::pop_jf_to(depth - 1); } + if let Some(entry) = CA_ACTIVE_FRAMES.with(|v| v.borrow_mut().pop()) { + CA_FRAME_POOL.with(|pool| pool.borrow_mut().push(entry)); + } 0 } @@ -464,22 +598,26 @@ pub struct WasmBackend { constants: majit_ir::VecMap, /// llmodel.py:64-69 self.vtable_offset. vtable_offset: Option, - /// `PYRE_WASM_CA` (default off): compile a self-recursive single-int - /// `CallAssemblerR` bridge into an in-module `call_indirect` into the source - /// loop's table slot (guest→guest recursion) instead of declining it to a - /// per-call host round-trip. Read from the process-global [`WASM_CA_ENABLED`] - /// at construction so flag-off is byte-identical. Slice 1 is correct only - /// under a huge nursery (the fresh callee frames are not GC-rooted; see the - /// CA arm in codegen). + /// `PYRE_WASM_CA` (default ON; `=0` kill switch): compile a self-recursive + /// single-int `CallAssemblerR` bridge into an in-module `call_indirect` + /// into the source loop's table slot (guest→guest recursion) instead of + /// declining it to a per-call host round-trip. Read from the + /// process-global [`WASM_CA_ENABLED`] at construction so flag-off is + /// byte-identical. Callee frames are GC-visible libc-jitframes + /// (per-frame `jf_gcmap`, jf-shadow-stack rooted; see + /// `wasm_jit_ca_alloc_frame`), so the arm is collection-safe at any + /// nursery size. wasm_ca_enabled: bool, } -/// Process-global toggle for the self-recursive CALL_ASSEMBLER arm. The wasm -/// guest has no environment (`std::env::var` always fails there), so the host -/// runner reads `PYRE_WASM_CA` and flips this through the `pyre_jit_set_wasm_ca` -/// export — mirroring how `pyre_jit_set_enable_bridges` plumbs the bridge tracer -/// flag. `WasmBackend::new` snapshots it. -static WASM_CA_ENABLED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); +/// Process-global toggle for the self-recursive CALL_ASSEMBLER arm, default +/// ON (without it the fib recursion shape round-trips through the host per +/// call — suite `fib_recursive` times out). The wasm guest has no environment +/// (`std::env::var` always fails there), so the host runner reads +/// `PYRE_WASM_CA` (`=0` disables) and flips this through the +/// `pyre_jit_set_wasm_ca` export — mirroring how `pyre_jit_set_enable_bridges` +/// plumbs the bridge tracer flag. `WasmBackend::new` snapshots it. +static WASM_CA_ENABLED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); /// Host entry point for the `pyre_jit_set_wasm_ca` export (and native tests). pub fn set_wasm_ca_enabled(enabled: bool) { @@ -494,6 +632,43 @@ pub fn wasm_ca_enabled() -> bool { WASM_CA_ENABLED.load(std::sync::atomic::Ordering::Relaxed) } +/// Process-global toggle for in-module bridge chaining, mirroring +/// `call_jit::WASM_BRIDGES_ENABLED` (the tracer-side flag the +/// `pyre_jit_set_enable_bridges` export sets; the export pushes it here too). +/// `execute_token` reads it to size every host frame's Ref-home region at +/// least `failguard::FRAME_REF_HOME_FLOOR` — the sizing the frame-fit accept +/// check in `compile_bridge` relies on — while keeping the flag-off frame +/// layout untouched. Default ON (`PYRE_WASM_ENABLE_BRIDGES=0` disables via +/// the runner). +static WASM_BRIDGES_ENABLED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(true); + +/// Host entry point for the `pyre_jit_set_enable_bridges` export (and tests). +pub fn set_wasm_bridges_enabled(enabled: bool) { + WASM_BRIDGES_ENABLED.store(enabled, std::sync::atomic::Ordering::Relaxed); +} + +/// Whether in-module bridge chaining is enabled (see `WASM_BRIDGES_ENABLED`). +pub fn wasm_bridges_enabled() -> bool { + WASM_BRIDGES_ENABLED.load(std::sync::atomic::Ordering::Relaxed) +} + +/// Inline nursery-bump allocation fast path (rewrite.py malloc fast path; +/// see `codegen::NurseryAllocParams`). Default ON; `PYRE_WASM_INLINE_ALLOC=0` +/// (plumbed by the runner through `pyre_jit_set_inline_alloc`) is the kill +/// switch — every `New*` then goes back through the allocation helper call. +static WASM_INLINE_ALLOC_ENABLED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(true); + +/// Toggle the inline nursery-bump fast path (kill switch plumbing). +pub fn set_wasm_inline_alloc_enabled(enabled: bool) { + WASM_INLINE_ALLOC_ENABLED.store(enabled, std::sync::atomic::Ordering::Relaxed); +} + +fn wasm_inline_alloc_enabled() -> bool { + WASM_INLINE_ALLOC_ENABLED.load(std::sync::atomic::Ordering::Relaxed) +} + /// GC type id of the `JitFrame`. The single registration authority is `eval.rs` /// (the type is registered there alongside the rest of the heap types, before /// `freeze_types`); it pushes the id here through `set_wasm_jitframe_tid`, @@ -510,6 +685,9 @@ pub fn set_wasm_jitframe_tid(id: u32) { WASM_JITFRAME_TID.store(id, std::sync::atomic::Ordering::Relaxed); } +// Only read on the wasm32 execute_token path (CA frame allocs use libc +// jitframes and no longer consume the tid). +#[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))] fn wasm_jitframe_tid() -> u32 { WASM_JITFRAME_TID.load(std::sync::atomic::Ordering::Relaxed) } @@ -867,25 +1045,21 @@ fn bridge_is_self_recursive_int_ca(ops: &[Op], source_loop_number: u64) -> bool /// enter: `frame[0]` holds the exit `fail_index`, `frame[1..]` the exit slots, /// and the pending-exception cell is captured with `jit_exc_take` exactly as /// `execute_token` does after a GuardNoException / GuardException exit. -/// `compiled_ptr` is the source loop's [`CompiledWasmLoop`] (baked into the -/// trace by `compile_bridge`) whose `fail_descrs` resolves the index — the -/// loop's own and its chained bridges' exits share that one array. -/// /// `pyre-jit`'s `call_jit::wasm_ca_resume_deopt` calls this, then drives the /// resulting `DeadFrame` through the same `get_latest_descr_arc` / /// `get_*_value` / `grab_exc_value` Backend path the host's outermost deopt /// handling uses, so the in-guest deopt completes identically. -pub fn dead_frame_from_ran_frame(compiled_ptr: usize, frame_ptr: usize) -> DeadFrame { - let compiled = unsafe { &*(compiled_ptr as *const CompiledWasmLoop) }; +/// +/// `frame[0]` resolves through the GLOBAL fail-index space +/// (`failguard::global_fail_descr`) — the exit may belong to a bridge chained +/// past the source loop. `_compiled_ptr` (the source loop's metadata address, +/// baked into the CA arm) is kept in the trace ABI but no longer consulted. +pub fn dead_frame_from_ran_frame(_compiled_ptr: usize, frame_ptr: usize) -> DeadFrame { let frame = frame_ptr as *const i64; let exc_value = jit_exc_take(); let fail_index = unsafe { *frame } as u32; - let fail_descr = compiled - .fail_descrs - .borrow() - .get(fail_index as usize) - .expect("invalid fail_index from in-guest CA callee frame") - .clone(); + let fail_descr = + global_fail_descr(fail_index).expect("invalid fail_index from in-guest CA callee frame"); let num_outputs = fail_descr.fail_arg_types.len(); let raw_values: Vec = (0..num_outputs) .map(|i| unsafe { *frame.add(1 + i) }) @@ -1036,6 +1210,10 @@ impl majit_backend::Backend for WasmBackend { let alloc_fn_ptr = wasm_jit_alloc as *const () as usize as i64; let alloc_array_fn_ptr = wasm_jit_alloc_array as *const () as usize as i64; let wb_fn_ptr = wasm_jit_write_barrier as *const () as usize as i64; + // Exit indices come from the global fail-index space so a cross-trace + // chain's `frame[0]` resolves regardless of which module wrote it + // (`failguard::FAIL_DESCR_REGISTRY`). + let fail_index_base = fail_descr_base(); let (wasm_bytes, guard_exits, num_ref_homes, bridge_cells_base, bridge_cells_owner) = codegen::build_wasm_module( inputargs, @@ -1047,8 +1225,11 @@ impl majit_backend::Backend for WasmBackend { alloc_fn_ptr, alloc_array_fn_ptr, wb_fn_ptr, - 0, // fail_index_base: a loop owns fail indices [0, num_guards) + nursery_alloc_params(ops).as_ref(), + fail_index_base, + true, // is_loop 0, // external_jump_slot: a loop's JUMP is a local back-edge `br` + 0, // external_jump_key: unused without an external JUMP codegen::CaParams::default(), // a loop never emits the CA arm )?; @@ -1065,6 +1246,7 @@ impl majit_backend::Backend for WasmBackend { }) }) .collect(); + register_fail_descrs(&fail_descrs); let max_output_slots = guard_exits .iter() @@ -1098,23 +1280,86 @@ impl majit_backend::Backend for WasmBackend { // Skip a LABEL whose descr is not loop-target-backed (`set_label_block_id` // would panic on a non-`AtomicU32` slot). let mut label_block_id: u32 = 0; - let mut last_label_num_args: usize = 0; + let mut label_descrs: Vec = Vec::new(); for op in ops.iter() { if op.opcode != majit_ir::OpCode::Label { continue; } + // Descr identity of each label, in ordinal order, so + // `compile_bridge` can resolve which of THIS loop's labels a + // closing JUMP targets by Arc identity (the JUMP and the LABEL + // share the descr). The stamped `label_block_id` alone cannot: a + // loop retraced into several specializations re-stamps a shared + // descr, and every specialization's start label carries ordinal + // 0 — a bridge targeting ANOTHER specialization's label would + // otherwise be mis-chained into this one. + label_descrs.push( + op.getdescr() + .map(|d| std::sync::Arc::as_ptr(&d) as *const () as usize) + .unwrap_or(0), + ); if let Some(descr) = op.getdescr() { if let Some(target) = descr.as_loop_target_descr() { target.set_label_block_id(label_block_id); } } - last_label_num_args = op.getarglist().len(); label_block_id += 1; } - // Ordinal of the LAST LABEL (the one codegen emits the `loop` at); 0 when - // there are no labels (then `has_preamble` is false and it is unused). - let last_label_block_id = label_block_id.saturating_sub(1); - let is_single_label_peeled = codegen::is_single_label_peeled(ops); + // Per-label resume metadata (ordinal order) for `compile_bridge`'s + // accept condition: a loop-closing bridge may resume at ANY label via + // the entry `br_table`, provided its JUMP arity matches that label's + // arg count and the label's args are the complete live set of the + // trace remainder. + let label_num_args = codegen::label_arg_counts(ops); + let label_resume_safe = codegen::label_resume_safety(ops); + // Per-guard, per-fail-arg induction-advance flags for + // `compile_bridge`'s livelock check (see `guard_fail_args_advanced`). + let guard_fail_arg_advanced = guard_fail_args_advanced(ops, &guard_exits); + + // Publish this loop's enterable labels so a loop-closing bridge from + // ANY loop can chain into them in-module (jump-to-existing-trace). A + // peeled loop's labels are each enterable through the entry br_table + // (key = ordinal + 1). A non-peeled loop has no dispatch: only its + // FIRST label is enterable — through the plain entry (key 0), whose + // input loader reads `num_inputs` positional slots — and only when + // the label's arity equals that (the standard loop shape, whose + // first label's args ARE the inputargs). + if has_preamble { + let last = label_descrs.len().saturating_sub(1); + for (j, &id) in label_descrs.iter().enumerate() { + if id == 0 { + continue; + } + publish_label_target( + id, + LabelTarget { + func_handle, + key: j as u32 + 1, + num_args: label_num_args[j], + resume_safe: label_resume_safe[j], + is_last_label: j == last, + num_ref_homes, + }, + ); + } + } else if let Some(&id) = label_descrs.first() { + if id != 0 && label_num_args.first() == Some(&inputargs.len()) { + publish_label_target( + id, + LabelTarget { + func_handle, + key: 0, + num_args: inputargs.len(), + resume_safe: true, + // No real ops precede a non-peeled loop's header, so + // an entry re-run lands at the header without any + // advancing segment — the livelock check applies. + is_last_label: true, + num_ref_homes, + }, + ); + } + } let compiled = CompiledWasmLoop { trace_id, @@ -1127,13 +1372,14 @@ impl majit_backend::Backend for WasmBackend { bridge_cells_base, num_guard_cells: guard_exits.len(), has_preamble, - is_single_label_peeled, - last_label_block_id, - last_label_num_args, + label_descrs, + guard_fail_arg_advanced, bridge_descr_ranges: std::cell::RefCell::new(Vec::new()), + chained_trace_meta: std::cell::RefCell::new(std::collections::HashMap::new()), _bridge_cells_owner: bridge_cells_owner, _bridge_owned_cells: std::cell::RefCell::new(Vec::new()), ca_bridge_ref_homes: std::cell::Cell::new(0), + ca_active: std::cell::Cell::new(false), }; token.compiled = Some(Box::new(compiled)); @@ -1206,20 +1452,17 @@ impl majit_backend::Backend for WasmBackend { // Scalars read from the source loop up front, so the immutable borrow of // `original_token` is released before the `&mut self` codegen calls. let ( - source_loop_trace_id, - source_cells_base, - source_num_cells, + source_guard, + source_is_direct, source_num_ref_homes, source_func_handle, source_has_preamble, - source_is_single_label_peeled, - source_last_label_block_id, - source_last_label_num_args, - base, source_max_output_slots, source_num_inputs, source_loop_finish_fi, source_compiled_ptr, + source_ca_active, + source_has_bridges, ) = { let source_loop = original_token .compiled @@ -1240,30 +1483,103 @@ impl majit_backend::Backend for WasmBackend { .find(|d| d.is_finish) .map(|d| d.fail_index) .unwrap_or(0); + // Resolve the failing guard's owning trace by the descr's + // `trace_id`: the source loop itself, or one of the bridges + // already chained onto it (`chained_trace_meta`) — a NESTED + // sub-bridge source. Either way the resolution yields the owning + // trace's guard-cell array, cell count, and the guard's + // per-fail-arg advance flags. `None` = foreign trace (declined + // below, diag 3). + let is_direct = source_trace_id == source_loop.trace_id; + let guard = if is_direct { + Some(( + source_loop.bridge_cells_base, + source_loop.num_guard_cells, + source_loop + .guard_fail_arg_advanced + .get(source_fail_index as usize) + .cloned() + .unwrap_or_default(), + )) + } else { + source_loop + .chained_trace_meta + .borrow() + .get(&source_trace_id) + .map(|m| { + ( + m.cells_base, + m.num_cells, + m.guard_fail_arg_advanced + .get(source_fail_index as usize) + .cloned() + .unwrap_or_default(), + ) + }) + }; ( - source_loop.trace_id, - source_loop.bridge_cells_base, - source_loop.num_guard_cells, + guard, + is_direct, source_loop.num_ref_homes, source_loop.func_handle, source_loop.has_preamble, - source_loop.is_single_label_peeled, - source_loop.last_label_block_id, - source_loop.last_label_num_args, - source_loop.fail_descrs.borrow().len() as u32, source_loop.max_output_slots, source_loop.num_inputs, loop_finish_fi, - // Address of the source loop's metadata, baked into the CA arm so - // `wasm_ca_resume_deopt` can resolve a deopted callee's - // `fail_descrs`. Same lifetime assumption as `source_func_handle` - // below: a recompile invalidates this bridge before the loop's - // `CompiledWasmLoop` is replaced, so the arm is unreachable with a - // stale pointer. + // Address of the source loop's metadata, baked into the CA arm + // (opaque cookie in the deopt-helper ABI; `frame[0]` resolution + // itself goes through the global fail-index space). Same + // lifetime assumption as `source_func_handle` below: a + // recompile invalidates this bridge before the loop's + // `CompiledWasmLoop` is replaced, so the arm is unreachable + // with a stale pointer. source_loop as *const CompiledWasmLoop as usize as u64, + source_loop.ca_active.get(), + !source_loop.bridge_descr_ranges.borrow().is_empty(), ) }; + // The failing guard must belong to the source loop or to a bridge + // already chained onto it, and its per-trace index must have a cell in + // that trace's array. A foreign descr has no cell to flip; decline so + // the metainterp keeps the correct interpreter fallback rather than + // installing an unreachable bridge module. + let Some((source_cells_base, source_num_cells, source_fail_arg_advanced)) = source_guard + else { + diag_bump(3); // declined: source guard's trace is not chained here + return Err(BackendError::Unsupported( + "wasm backend: bridge source guard is not a direct loop guard".into(), + )); + }; + if source_fail_index as usize >= source_num_cells { + diag_bump(3); + return Err(BackendError::Unsupported( + "wasm backend: bridge source guard index has no dispatch cell".into(), + )); + } + // The CA arm bakes source-LOOP metadata (finish index, compiled ptr); + // restrict it to direct loop guards. A CA-shaped bridge on a nested + // guard then fails codegen's CALL_ASSEMBLER handling — a deterministic + // decline. + let allow_ca = allow_ca && source_is_direct; + // The CA arm and further bridge chaining do not compose yet: a chained + // bridge deopting inside the CA recursion trips a resume seam that + // reads a clobbered class (wrong output on suite + // `recursion_memo_branch` / `generator_tree_recursion`; each mechanism + // alone is correct). Until that seam is fixed, a recursion gets ONE of + // the two: the CA lift only for a loop with no chained bridges yet, + // and no further chaining once the CA cell is live — the declined + // guard falls back to host round-trips, which handle it correctly. + let allow_ca = allow_ca && !source_has_bridges; + if !allow_ca && source_ca_active { + diag_bump(14); // declined: source recursion is CA-active + return Err(BackendError::Unsupported( + "wasm backend: source recursion is CA-active; further bridge \ + chaining declined" + .into(), + )); + } + // A loop-closing bridge (terminal JUMP, no local LABEL) re-enters the // source loop through `source_func_handle` — the function entry. For a // peeled source loop, entering at the function entry re-runs the preamble @@ -1271,14 +1587,17 @@ impl majit_backend::Backend for WasmBackend { // instead of resuming at the LABEL, so the induction variable never // advances: an infinite loop (the wasm chaining hang on nbody / fannkuch). // - // A SINGLE-label peeled loop carries the resume-at-LABEL dispatch: the - // loop-closing JUMP arm sets the frame dispatch key, so re-entering - // through `source_func_handle` skips the preamble and resumes at the - // LABEL — chaining stays in-module. Only a MULTI-label peeled loop (the - // br_table form is a follow-up) still re-runs its preamble, so decline - // it; the guard then falls back to blackhole resume and - // `declined_bridge_guards` stops the metainterp re-tracing it. Non-peeled - // loops (entry == LABEL) re-enter correctly and keep chaining. + // A peeled loop carries the resume-at-LABEL dispatch: the loop-closing + // JUMP arm sets the frame dispatch key to `target label ordinal + 1`, + // so re-entering through `source_func_handle` `br_table`s to that + // label's resume loader — chaining stays in-module. The bridge is + // accepted when its JUMP's target label is recoverable from the descr, + // the arities match, and the label's args are the complete live set of + // the trace remainder (`label_resume_safe`); otherwise decline — the + // guard then falls back to blackhole resume and + // `declined_bridge_guards` stops the metainterp re-tracing it. + // Non-peeled loops (entry == LABEL) re-enter correctly and keep + // chaining. let bridge_is_loop_closing = { let has_label = ops.iter().any(|op| op.opcode == majit_ir::OpCode::Label); let has_jump = ops.iter().any(|op| op.opcode == majit_ir::OpCode::Jump); @@ -1290,43 +1609,77 @@ impl majit_backend::Backend for WasmBackend { if source_has_preamble { diag_bump(7); // source loop has preamble } - if bridge_is_loop_closing && source_has_preamble { - // The peeled source resumes at its LAST label (where the `loop` is) - // via the resume-at-LABEL dispatch. Accept this bridge only if its - // terminal JUMP re-enters at that last label: a single-label source - // has only that one label, so accept directly; a multi-label source - // is accepted only when the JUMP's target ordinal — recovered from - // its descr, which the source LABEL stamped with `set_label_block_id` - // (shared by Arc identity) — equals the source's last label and the - // arities match. Any other target (a non-last label) needs the - // deferred br_table, so decline → blackhole resume, and - // `declined_bridge_guards` stops the metainterp re-tracing it. - let resumes_at_last_label = source_is_single_label_peeled || { - let closing_jump = ops - .iter() - .rev() - .find(|op| op.opcode == majit_ir::OpCode::Jump); - let target_ord = closing_jump - .and_then(|j| j.getdescr()) - .and_then(|d| d.as_loop_target_descr().map(|t| t.label_block_id())); - let arity = closing_jump.map_or(0, |j| j.getarglist().len()); - // Sub-breakdown of why a multi-label bridge does not resume at the - // last label (diagnostic: distinguishes the S2-needing non-last - // case from a stripped descr or an arity mismatch). - match target_ord { - None => diag_bump(8), // descr stripped - Some(k) if k != source_last_label_block_id => diag_bump(9), // non-last label - _ if arity != source_last_label_num_args => diag_bump(10), // arity mismatch - _ => {} + // Resolve the terminal JUMP's target label BY DESCR IDENTITY through + // the `LABEL_TARGETS` registry — the JUMP and its target LABEL share + // the loop-target descr Arc, and every compiled loop published its + // enterable labels there. The stamped `label_block_id` ordinal is NOT + // identity: a retraced loop has several sibling specializations whose + // start labels all carry ordinal 0, and a bridge legitimately closes + // into a SIBLING (jump-to-existing-trace) — the registry resolves the + // owning module's table slot and resume key, so the tail call chains + // into the RIGHT loop, own or sibling. Decline when the target is + // unpublished (descr stripped, or its loop declined/was dropped), the + // JUMP arity differs from the label's arg count (the resume loader + // reads exactly that many positional frame slots), the label's args + // are not the complete live set of the target trace's remainder + // (`resume_safe` — resuming there would read a null local), or the + // target loop's Ref-home region exceeds what the chain's entry frame + // is guaranteed to carry (`max(source homes, FRAME_REF_HOME_FLOOR)` — + // the frame `execute_token` sized for the loop the chain entered + // through; requiring target ≤ that bound keeps every hop within the + // entry frame by induction, since the entry frame itself is sized to + // at least the floor). Ref homes are the only variable requirement: + // value slots are bounded by codegen's `CALL_AREA_FIRST_SLOT` decline, + // below the constant `MIN_FRAME_BYTES / 8` value region every host + // frame carries. A declined guard falls back to blackhole resume and + // `declined_bridge_guards` stops the metainterp re-tracing it. + let mut external_jump_key: u32 = 0; + let mut external_jump_slot: u32 = source_func_handle; + let mut resumes_at_loop_header = false; + if bridge_is_loop_closing { + let closing_jump = ops + .iter() + .rev() + .find(|op| op.opcode == majit_ir::OpCode::Jump); + let target = closing_jump + .and_then(|j| j.getdescr()) + .map(|d| std::sync::Arc::as_ptr(&d) as *const () as usize) + .filter(|id| *id != 0) + .and_then(label_target); + let arity = closing_jump.map_or(0, |j| j.getarglist().len()); + let accepted_target = match target { + // Descr stripped, or the target label was never published. + None => { + diag_bump(8); + false + } + Some(t) if arity != t.num_args => { + diag_bump(10); // arity mismatch + false + } + Some(t) if !t.resume_safe => { + diag_bump(9); // label args not the full live set + false + } + Some(t) + if t.num_ref_homes + > source_num_ref_homes.max(failguard::FRAME_REF_HOME_FLOOR) => + { + diag_bump(4); // target Ref homes exceed the entry frame's bound + false + } + Some(t) => { + external_jump_key = t.key; + external_jump_slot = t.func_handle; + resumes_at_loop_header = t.is_last_label; + true } - matches!(target_ord, Some(k) if k == source_last_label_block_id) - && arity == source_last_label_num_args }; - if !resumes_at_last_label { - diag_bump(2); // declined: peeled source, JUMP not resuming at last label + if !accepted_target { + diag_bump(2); // declined: JUMP target not chainable return Err(BackendError::Unsupported( - "wasm backend: loop-closing bridge re-enters a peeled loop at a \ - non-last label (resume-at-LABEL br_table deferred)" + "wasm backend: loop-closing bridge JUMP target is not a \ + chainable published label" .into(), )); } @@ -1345,7 +1698,29 @@ impl majit_backend::Backend for WasmBackend { // to blackhole resume and `declined_bridge_guards` stops the metainterp // re-tracing it. A genuinely advancing loop-closing bridge (an `i += 1` // counter feeding a JUMP arg) passes and keeps chaining. - if bridge_is_loop_closing { + // + // The check only concerns a bridge that lands directly AT the loop + // header (the target's last label, or the entry of a non-peeled + // loop): only then can the guard re-fail on byte-identical state. A + // resume at an EARLIER label executes the segment between that label + // and the header — the peeled iteration — which advances the state + // before the loop re-runs, so no advance is required of the bridge + // itself. + if bridge_is_loop_closing && resumes_at_loop_header { + // Bridge input position `k` reads frame slot `k`, where the source + // guard spilled its k-th fail arg — so an `InputArg` JUMP arg is a + // verbatim reload of source fail arg `k`. The advance for such an + // arg may have happened in the SOURCE loop's body before the guard + // (an `i += 1` preceding the failing branch): the source recorded + // per-fail-arg whether the value was produced by induction- + // advancing arithmetic within the failing iteration + // (`guard_fail_arg_advanced`), so consult that alongside the + // in-bridge producers. + let input_pos: std::collections::HashMap = inputargs + .iter() + .enumerate() + .map(|(k, ia)| (ia.index, k)) + .collect(); let advances = ops .iter() .rev() @@ -1355,10 +1730,37 @@ impl majit_backend::Backend for WasmBackend { majit_ir::operand::Operand::Op(producer) => { is_inductive_arith(producer.opcode) } + majit_ir::operand::Operand::InputArg(ia) => { + input_pos.get(&ia.index).is_some_and(|&k| { + source_fail_arg_advanced.get(k).copied().unwrap_or(false) + }) + } _ => false, }) }); - if !advances { + // Loop state carried on the HEAP (a permutation array flipped via + // setarrayitem, an object field bumped via setfield, a residual + // call's arbitrary effects) advances the cycle without any JUMP + // arg showing inductive arithmetic. The shield only exists to + // refuse PROVABLY static bridges, so any state-mutating op counts + // as an advance. + let mutates_heap = ops.iter().any(|op| { + use majit_ir::OpCode::*; + op.opcode.is_call() + || matches!( + op.opcode, + SetfieldGc + | SetfieldRaw + | SetarrayitemGc + | SetarrayitemRaw + | GcStore + | GcStoreIndexed + | RawStore + | Strsetitem + | Unicodesetitem + ) + }); + if !advances && !mutates_heap { diag_bump(11); // declined: loop-closing bridge advances no loop-carried value return Err(BackendError::Unsupported( "wasm backend: loop-closing bridge advances no loop-carried value \ @@ -1368,20 +1770,6 @@ impl majit_backend::Backend for WasmBackend { } } - // This simple chaining handles a bridge attached directly to one of the - // source loop's own guards (the common loop-exit continuation). A nested - // bridge (source guard living in another bridge) or a foreign descr has - // no cell in this loop's array; decline so the metainterp keeps the - // correct interpreter fallback rather than installing an unreachable - // bridge module. - if source_trace_id != source_loop_trace_id || source_fail_index as usize >= source_num_cells - { - diag_bump(3); // declined: not a direct loop guard - return Err(BackendError::Unsupported( - "wasm backend: bridge source guard is not a direct loop guard".into(), - )); - } - self.collect_constants_from_ops(ops); let trace_id = self.trace_counter; self.trace_counter += 1; @@ -1407,8 +1795,24 @@ impl majit_backend::Backend for WasmBackend { // region and a minor collection mid-recursion reclaims it, leaving a later // deopt to read zeroed nursery memory. `count_ref_homes` matches the // `num_ref_homes` `build_wasm_module` returns below. + // + // With bridge chaining on, the recursion can also chain into NESTED + // bridges (and sibling loops via loop-closing tail calls) while running + // ON a CA callee frame — and those were accepted against the + // `FRAME_REF_HOME_FLOOR` bound `execute_token` guarantees for HOST + // frames. The callee frame must give the same guarantee, or a chained + // bridge homes/reads Ref slots past the frame's sized (and gcmap-walked) + // region — wrong-value corruption (suite `recursion_memo_branch` / + // `generator_tree_recursion`). Mirror `execute_token`'s `chain_floor`. + let chain_floor = if wasm_bridges_enabled() { + failguard::FRAME_REF_HOME_FLOOR + } else { + 0 + }; let ca_ref_homes = if allow_ca { - source_num_ref_homes.max(codegen::count_ref_homes(inputargs, ops)) + source_num_ref_homes + .max(codegen::count_ref_homes(inputargs, ops)) + .max(chain_floor) } else { source_num_ref_homes }; @@ -1436,7 +1840,10 @@ impl majit_backend::Backend for WasmBackend { codegen::CaParams::default() }; - let (wasm_bytes, guard_exits, num_ref_homes, _bridge_cells_base, bridge_cells_owner) = + // This bridge's exit indices come from the global fail-index space, + // like every trace's (`failguard::FAIL_DESCR_REGISTRY`). + let base = fail_descr_base(); + let (wasm_bytes, guard_exits, num_ref_homes, bridge_cells_base, bridge_cells_owner) = codegen::build_wasm_module( inputargs, ops, @@ -1447,17 +1854,24 @@ impl majit_backend::Backend for WasmBackend { alloc_fn_ptr, alloc_array_fn_ptr, wb_fn_ptr, + nursery_alloc_params(ops).as_ref(), base, - // A loop-closing bridge's terminal JUMP re-enters the source loop - // through its table slot via a tail call. - source_func_handle, + false, // is_loop + // A loop-closing bridge's terminal JUMP re-enters the target + // loop (own or sibling, resolved via `LABEL_TARGETS`) through + // its table slot via a tail call, resuming at the label + // `external_jump_key` selects. + external_jump_slot, + external_jump_key, ca_params, )?; - // The bridge runs in the source loop's fixed-size frame, so it must not - // address more Ref-home slots than the loop reserved. If it would, - // decline: the host round-trip path allocates a frame sized for the - // bridge. The bridge's value/output slots need no separate bound check: + // The bridge runs in the chain's entry frame, so it must not address + // more Ref-home slots than that frame is guaranteed to carry — + // `max(source loop's homes, FRAME_REF_HOME_FLOOR)`, the same inductive + // bound as the JUMP-target check above. If it would, decline: the host + // round-trip path allocates a frame sized for the bridge. + // The bridge's value/output slots need no separate bound check: // `build_wasm_module` already declined this bridge (above, via `?`) if // its value slots reach `CALL_AREA_FIRST_SLOT` (codegen.rs), and // `execute_token` floors the frame at `MIN_FRAME_BYTES/8` slots — which @@ -1470,11 +1884,11 @@ impl majit_backend::Backend for WasmBackend { // (`callee_frame_bytes`), and the outermost call runs in the host entry // frame `F0`, which `execute_token` widens via `ca_bridge_ref_homes` // (set below). So its home writes never overflow. - if !allow_ca && num_ref_homes > source_num_ref_homes { + if !allow_ca && num_ref_homes > source_num_ref_homes.max(failguard::FRAME_REF_HOME_FLOOR) { diag_bump(4); // declined: ref-home overflow return Err(BackendError::Unsupported(format!( - "wasm backend: bridge needs {num_ref_homes} ref homes, source loop has \ - {source_num_ref_homes}" + "wasm backend: bridge needs {num_ref_homes} ref homes, entry frame bound is \ + max({source_num_ref_homes}, floor)" ))); } @@ -1491,6 +1905,7 @@ impl majit_backend::Backend for WasmBackend { }) }) .collect(); + register_fail_descrs(&bridge_descrs); // Register the bridge module into the shared table, then publish its // descrs and flip the source guard's cell. Order matters: the descrs @@ -1522,11 +1937,23 @@ impl majit_backend::Backend for WasmBackend { let start = descrs.len(); descrs.extend(bridge_descrs); source_loop.bridge_descr_ranges.borrow_mut().push(( + source_trace_id, source_fail_index, start, count, )); } + // Publish this bridge's own guard-dispatch metadata so a hot guard + // INSIDE it can chain a nested sub-bridge (same resolution the + // loop's own guards get, keyed by this bridge's trace_id). + source_loop.chained_trace_meta.borrow_mut().insert( + trace_id, + ChainedTraceMeta { + cells_base: bridge_cells_base, + num_cells: guard_exits.len(), + guard_fail_arg_advanced: guard_fail_args_advanced(ops, &guard_exits), + }, + ); // The bridge module lives as long as this source loop, so hand its // own cell array (if any) to the loop, freed when the loop drops. if let Some(owner) = bridge_cells_owner { @@ -1539,6 +1966,9 @@ impl majit_backend::Backend for WasmBackend { if allow_ca { let prev = source_loop.ca_bridge_ref_homes.get(); source_loop.ca_bridge_ref_homes.set(prev.max(num_ref_homes)); + // Freeze this recursion to the CA mechanism: no further bridge + // chains here (see the decline above the codegen call). + source_loop.ca_active.set(true); } } @@ -1659,7 +2089,7 @@ impl majit_backend::Backend for WasmBackend { fn compiled_bridge_fail_descr_layouts( &self, original_token: &JitCellToken, - _source_trace_id: u64, + source_trace_id: u64, source_fail_index: u32, ) -> Option> { let compiled = original_token @@ -1672,8 +2102,8 @@ impl majit_backend::Backend for WasmBackend { .borrow() .iter() .rev() - .find(|r| r.0 == source_fail_index) - .map(|&(_, start, count)| (start, count))?; + .find(|r| r.0 == source_trace_id && r.1 == source_fail_index) + .map(|&(_, _, start, count)| (start, count))?; let descrs = compiled.fail_descrs.borrow(); let layouts = descrs .get(start..start + count)? @@ -1714,7 +2144,7 @@ impl majit_backend::Backend for WasmBackend { fn store_bridge_guard_hashes( &self, token: &JitCellToken, - _source_trace_id: u64, + source_trace_id: u64, source_fail_index: u32, hashes: &[u64], ) { @@ -1730,8 +2160,8 @@ impl majit_backend::Backend for WasmBackend { .borrow() .iter() .rev() - .find(|r| r.0 == source_fail_index) - .map(|&(_, start, count)| (start, count)) + .find(|r| r.0 == source_trace_id && r.1 == source_fail_index) + .map(|&(_, _, start, count)| (start, count)) else { return; }; @@ -1772,9 +2202,19 @@ impl majit_backend::Backend for WasmBackend { // outermost call in this frame and may home more Refs than the loop, so // size for the LARGER of the two (`ca_bridge_ref_homes`); the extra slots // are zeroed and GC-rooted below exactly like the loop's own homes. + // With bridge chaining on, a cross-trace tail call can land in a loop + // or bridge homing more Refs than this one, so also size at least + // `FRAME_REF_HOME_FLOOR` — the bound `compile_bridge`'s frame-fit + // accept checks rely on. Flag-off keeps the exact old sizing. + let chain_floor = if wasm_bridges_enabled() { + failguard::FRAME_REF_HOME_FLOOR + } else { + 0 + }; let eff_ref_homes = compiled .num_ref_homes - .max(compiled.ca_bridge_ref_homes.get()); + .max(compiled.ca_bridge_ref_homes.get()) + .max(chain_floor); let frame_size = base_slots + 1 + eff_ref_homes; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { @@ -1837,12 +2277,11 @@ impl majit_backend::Backend for WasmBackend { let exc_value = jit_exc_take(); let fail_index = unsafe { *(items_base as *const i64) } as u32; - let fail_descr = compiled - .fail_descrs - .borrow() - .get(fail_index as usize) - .expect("invalid fail_index from compiled wasm") - .clone(); + // Global fail-index space: a cross-trace chain may exit through + // a sibling loop's guard, so `frame[0]` never resolves against + // this loop's own `fail_descrs`. + let fail_descr = + global_fail_descr(fail_index).expect("invalid fail_index from compiled wasm"); let num_outputs = fail_descr.fail_arg_types.len(); let raw_values: Vec = (0..num_outputs) .map(|i| unsafe { *((items_base + fsb + i * 8) as *const i64) }) @@ -1891,12 +2330,9 @@ impl majit_backend::Backend for WasmBackend { } let exc_value = jit_exc_take(); let fail_index = frame[0] as u32; - let fail_descr = compiled - .fail_descrs - .borrow() - .get(fail_index as usize) - .expect("invalid fail_index from compiled wasm") - .clone(); + // Global fail-index space (see the CA-path resolution above). + let fail_descr = + global_fail_descr(fail_index).expect("invalid fail_index from compiled wasm"); let num_outputs = fail_descr.fail_arg_types.len(); let raw_values: Vec = (0..num_outputs).map(|i| frame[1 + i]).collect(); DeadFrame { diff --git a/majit/majit-backend-wasm/tests/codegen_test.rs b/majit/majit-backend-wasm/tests/codegen_test.rs index a2b28ca6f71..4d08583e861 100644 --- a/majit/majit-backend-wasm/tests/codegen_test.rs +++ b/majit/majit-backend-wasm/tests/codegen_test.rs @@ -52,8 +52,11 @@ fn test_empty_trace() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -113,8 +116,11 @@ fn test_int_add_loop() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -176,8 +182,11 @@ fn test_float_ops() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -215,8 +224,11 @@ fn test_call_generates_import() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -311,8 +323,11 @@ fn test_guard_types() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -357,8 +372,11 @@ fn test_exception_guards() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -399,8 +417,11 @@ fn test_guard_gc_type_uses_immediate_typeid() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -463,8 +484,11 @@ fn test_guard_is_object_lowers_to_typeinfo_test() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed when supports_guard_gc_type=true"); @@ -515,8 +539,11 @@ fn test_guard_subclass_lowers_to_subclassrange_check() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed when supports_guard_gc_type=true"); @@ -534,8 +561,11 @@ fn test_guard_subclass_lowers_to_subclassrange_check() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed when vtable_offset is set"); @@ -602,8 +632,11 @@ fn test_sameas_and_conversions() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -656,8 +689,11 @@ fn test_overflow_ops() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -714,8 +750,11 @@ fn test_single_label_peeled_loop_validates() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); @@ -779,8 +818,11 @@ fn test_multi_label_peeled_resumes_at_last_label_validates() { 0, 0, 0, - 0, // fail_index_base - 0, // external_jump_slot + None, // nursery + 0, // fail_index_base + true, // is_loop + 0, // external_jump_slot + 0, // external_jump_key codegen::CaParams::default(), ) .expect("wasm codegen should succeed"); diff --git a/majit/majit-gc/src/collector.rs b/majit/majit-gc/src/collector.rs index 814192d5901..4238eaf5a97 100644 --- a/majit/majit-gc/src/collector.rs +++ b/majit/majit-gc/src/collector.rs @@ -2806,6 +2806,17 @@ impl GcAllocator for MiniMarkGC { self.alloc_in_oldgen(type_id, total_size) } + fn collection_counts(&self) -> (usize, usize) { + (self.minor_collections, self.major_collections) + } + + fn type_alloc_is_plain(&self, type_id: u32) -> bool { + (type_id as usize) < self.types.len() && { + let info = self.types.get(type_id); + info.destructor.is_none() && !info.is_weakref + } + } + fn is_managed_heap_object(&self, addr: usize) -> bool { self.is_valid_gc_object(addr) && (self.nursery.contains(addr) || self.oldgen.contains(addr)) } diff --git a/majit/majit-gc/src/lib.rs b/majit/majit-gc/src/lib.rs index d4325ae328f..47036a79517 100644 --- a/majit/majit-gc/src/lib.rs +++ b/majit/majit-gc/src/lib.rs @@ -57,6 +57,14 @@ pub mod flags { pub const DUMMY: u64 = 1 << 12; } +/// True when the `gc_stress` test feature is compiled in: every allocation +/// may then run a full collection inside `alloc_with_type`, so JIT fast +/// paths that bypass it (inline nursery bump) must stay disabled or the +/// stress coverage silently shrinks to non-JIT allocations. +pub fn gc_stress_enabled() -> bool { + cfg!(feature = "gc_stress") +} + /// Write barrier descriptor — information the JIT needs to emit write barrier checks. /// /// From rpython/jit/backend/llsupport/gc.py WriteBarrierDescr. @@ -374,6 +382,23 @@ pub trait GcAllocator: Send { (0, 0) } + /// Diagnostic only: `(minor_collections, major_collections)` run so far. + /// Used to attribute run time to collection cadence (e.g. old-gen churn + /// driving repeated majors). Default `(0, 0)` for stub allocators. + fn collection_counts(&self) -> (usize, usize) { + (0, 0) + } + + /// Whether a JIT inline nursery bump of `type_id` is equivalent to + /// `alloc_with_type`'s fast path: the type registers no destructor and is + /// not a weakref (either would need a side-list push at allocation, i.e. + /// the slow path). Mirrors rewrite.py's malloc fast-path eligibility + /// (types with finalizers/weakrefs keep the call). Default `false` so + /// stub allocators keep the helper path. + fn type_alloc_is_plain(&self, _type_id: u32) -> bool { + false + } + /// Look up the fixed-object size for a registered GC type. /// /// RPython parity: this matches `cpu.bh_new(typedescr)` reading diff --git a/majit/majit-metainterp/src/call_descr.rs b/majit/majit-metainterp/src/call_descr.rs index ad79f212ddc..041053e6488 100644 --- a/majit/majit-metainterp/src/call_descr.rs +++ b/majit/majit-metainterp/src/call_descr.rs @@ -545,6 +545,37 @@ pub fn make_call_descr_with_effect( arg_types: &[Type], result_type: Type, effect_info: EffectInfo, +) -> DescrRef { + let (result_signed, result_size) = result_metadata(result_type); + make_call_descr_sized( + arg_types, + result_type, + result_signed, + result_size, + effect_info, + ) +} + +/// [`make_call_descr_with_effect`] variant for hand-written `extern "C"` +/// helpers recorded as void residuals whose C signature actually RETURNS a +/// dummy machine word (`fn(i64×n) -> i64`, value ignored). The descr keeps +/// `result_type = Void` (the recorded op is `CallN` with no result box) but +/// carries `result_size = 8` where a plain void descr carries 0, so a +/// backend that emits signature-exact direct calls (wasm `call_indirect`) +/// can select the `(i64×n) -> i64` type and drop the result. Reflective +/// dispatch paths never read `result_size` for a void result, and the field +/// participates in the interning key, so word-ABI descrs never collapse +/// with plain void descrs of the same shape. +pub fn make_call_descr_void_word_abi(arg_types: &[Type], effect_info: EffectInfo) -> DescrRef { + make_call_descr_sized(arg_types, Type::Void, false, 8, effect_info) +} + +fn make_call_descr_sized( + arg_types: &[Type], + result_type: Type, + result_signed: bool, + result_size: usize, + effect_info: EffectInfo, ) -> DescrRef { // `effectinfo.py:182-184` invariant: no new `EffectInfo` may be // constructed after `compute_bitstrings` has run; PyPy enforces this @@ -569,7 +600,6 @@ pub fn make_call_descr_with_effect( runs (codewriter setup phase).\n effect_info: {effect_info:?}" ); } - let (result_signed, result_size) = result_metadata(result_type); // effectinfo.py:144-146: `if tgt_func: key += (object(),) # don't // care about caching in this case` — release-gil targets bypass the // EffectInfo._cache via a fresh object() key. The call descr still diff --git a/majit/majit-metainterp/src/history.rs b/majit/majit-metainterp/src/history.rs index 07113b8e334..81c39629ac8 100644 --- a/majit/majit-metainterp/src/history.rs +++ b/majit/majit-metainterp/src/history.rs @@ -3048,19 +3048,32 @@ impl TraceCtx { arg_types: &[Type], ret_type: Type, ) -> OpRef { - let func_ref = OpRef::const_int(func_ptr as usize as i64); let descr = crate::call_descr::make_call_descr_for_opcode(opcode, arg_types, ret_type); + self.record_call_with_descr(opcode, func_ptr, args, descr) + } + + /// Shared record tail for the `call_*_typed` family: prepend the funcbox, + /// invalidate heap caches, record the op with `descr`. + /// + /// pyjitpl.py:2683-2684 `_record_helper_varargs` parity: + /// `heapcache.invalidate_caches_varargs(...)` runs BEFORE + /// `self.history.record(...)`. Routes every CALL family record + /// through `invalidate_caches_varargs` so the elidable / + /// loopinvariant / arraycopy / arraymove fast-paths inside + /// `clear_caches_varargs` (heapcache.py:341-376) run exactly once + /// per call. The previous escape-only path + /// (`_escape_argboxes + invalidate_caches_for_escaped`) skipped + /// those branches. + fn record_call_with_descr( + &mut self, + opcode: OpCode, + func_ptr: *const (), + args: &[OpRef], + descr: majit_ir::DescrRef, + ) -> OpRef { + let func_ref = OpRef::const_int(func_ptr as usize as i64); let mut call_args = vec![func_ref]; call_args.extend_from_slice(args); - // pyjitpl.py:2683-2684 `_record_helper_varargs` parity: - // `heapcache.invalidate_caches_varargs(...)` runs BEFORE - // `self.history.record(...)`. Routes every CALL family record - // through `invalidate_caches_varargs` so the elidable / - // loopinvariant / arraycopy / arraymove fast-paths inside - // `clear_caches_varargs` (heapcache.py:341-376) run exactly once - // per call. The previous escape-only path - // (`_escape_argboxes + invalidate_caches_for_escaped`) skipped - // those branches. if let Some(call_descr) = descr.as_call_descr() { let oracle: &dyn crate::heapcache::SameConstantOracle = &crate::history::ConstOprefOracle; @@ -3084,6 +3097,30 @@ impl TraceCtx { let _ = self.call_typed(OpCode::CallN, func_ptr, args, arg_types, Type::Void); } + /// [`call_void_typed`] for hand-written `extern "C"` helpers whose C + /// signature returns a dummy machine word (`-> i64`, value ignored). + /// Records the same `CallN` op through a descr that carries the true + /// callee ABI (`make_call_descr_void_word_abi`) so a signature-exact + /// backend lowering can call it directly. + /// + /// `effect_info` is caller-supplied because these helpers WRITE the + /// heap (namespace dict cells, list storage): the opcode default + /// (`default_effect_info`, empty write sets) would tell the + /// optimizer the call touches no tracked field, letting optheap CSE + /// a getfield across the call and read a stale value. An + /// unanalyzed external writer follows `graphanalyze.py:60 + /// analyze_external_call` top: `EffectInfo::MOST_GENERAL`. + pub fn call_void_typed_word_abi( + &mut self, + func_ptr: *const (), + args: &[OpRef], + arg_types: &[Type], + effect_info: majit_ir::EffectInfo, + ) { + let descr = crate::call_descr::make_call_descr_void_word_abi(arg_types, effect_info); + let _ = self.record_call_with_descr(OpCode::CallN, func_ptr, args, descr); + } + /// `call_typed` variant that preserves the caller-supplied `EffectInfo` /// instead of re-deriving the default for the opcode. Mirrors /// `pyjitpl.py:1995-2068 do_residual_call` parity: PyPy passes the @@ -3100,30 +3137,9 @@ impl TraceCtx { ret_type: Type, effect_info: majit_ir::EffectInfo, ) -> OpRef { - let func_ref = OpRef::const_int(func_ptr as usize as i64); let descr = crate::call_descr::make_call_descr_with_effect(arg_types, ret_type, effect_info); - let mut call_args = vec![func_ref]; - call_args.extend_from_slice(args); - // pyjitpl.py:2683-2684 `_record_helper_varargs` parity (see - // `call_typed` for the full rationale): invalidate before record. - if let Some(call_descr) = descr.as_call_descr() { - let oracle: &dyn crate::heapcache::SameConstantOracle = - &crate::history::ConstOprefOracle; - let const_value = |opref: OpRef| match opref.inline_const_to_value() { - Some(majit_ir::Value::Int(n)) => Some(n), - _ => None, - }; - self.heap_cache.invalidate_caches_varargs( - opcode, - Some(call_descr.get_extra_info()), - &call_args, - oracle, - const_value, - ); - } - self.recorder - .record_op_with_descr(opcode, &call_args, descr.clone()) + self.record_call_with_descr(opcode, func_ptr, args, descr) } pub fn call_void_typed_with_effect( diff --git a/majit/majit-metainterp/src/jitcode/assembler.rs b/majit/majit-metainterp/src/jitcode/assembler.rs index 301fd451905..ab822ecc24a 100644 --- a/majit/majit-metainterp/src/jitcode/assembler.rs +++ b/majit/majit-metainterp/src/jitcode/assembler.rs @@ -438,7 +438,7 @@ impl JitCodeBuilder { let descr = self.add_bh_descr(CanonicalBhDescr::Size { size, type_id, - vtable, + vtable: vtable as u64, owner: String::new(), all_fielddescrs: Vec::new(), is_gc_managed: true, diff --git a/majit/majit-metainterp/src/jitdriver.rs b/majit/majit-metainterp/src/jitdriver.rs index 07051c492c6..60998c7e6f5 100644 --- a/majit/majit-metainterp/src/jitdriver.rs +++ b/majit/majit-metainterp/src/jitdriver.rs @@ -4201,25 +4201,30 @@ impl JitDriver { raw_fail_values: &[i64], resume_pc: usize, ) -> bool { + majit_metainterp::mc_diag_bump(12); // start_bridge_tracing entered // compile.py:725-729 `_trace_and_compile_from_bridge` raises // `compile.giveup()` when the descr's owning JitCellToken weakref // is dead (memmgr-evicted). Pyre signals the same outcome by // returning false; also returns false if the descr is not a // FailDescr at all (e.g. a synthetic terminal-exit Descr). let Some(descr_fd) = descr_arc.as_fail_descr() else { + majit_metainterp::mc_diag_bump(13); // sbt early: descr not FailDescr return false; }; let Some(jct) = majit_backend::descr_owning_jct(descr_fd) else { + majit_metainterp::mc_diag_bump(14); // sbt early: no owning jct return false; }; let green_key = jct.green_key; let trace_id = descr_fd.trace_id(); let fail_index = descr_fd.fail_index_per_trace(); let Some(_loop_meta) = self.meta.get_compiled_meta(green_key).cloned() else { + majit_metainterp::mc_diag_bump(15); // sbt early: no compiled_meta return false; }; if !state.can_trace() { + majit_metainterp::mc_diag_bump(16); // sbt early: !can_trace return false; } @@ -4236,6 +4241,7 @@ impl JitDriver { // trace_id, fail_index)` reverse lookup. let fail_arg_count = descr_fd.fail_arg_types().len(); let Some(frontend_fail_values) = raw_fail_values.get(..fail_arg_count) else { + majit_metainterp::mc_diag_bump(17); // sbt early: fail_values too short return false; }; diff --git a/majit/majit-metainterp/src/lib.rs b/majit/majit-metainterp/src/lib.rs index 0e2dfb93d06..a71661e28e1 100644 --- a/majit/majit-metainterp/src/lib.rs +++ b/majit/majit-metainterp/src/lib.rs @@ -412,12 +412,18 @@ pub fn register_stack_almost_full_hook(f: fn() -> bool) { /// Diagnostic-only guard-failure → bridge-trace gate tallies, read out via /// the `pyre_jit_mc_diag` guest export. Index legend: 0 = must_compile_with_values /// entered, 1 = declined_bridge_guards short-circuit, 2 = descr_addr==0 skip, -/// 3 = status-busy skip, 4 = jitcounter FIRED (true), 5 = reserved (unused), -/// 6 = start_retrace_from_guard entered, 7 = start_retrace bailed (source loop -/// evicted: compiled_loops miss). -pub static MC_DIAG: [std::sync::atomic::AtomicU64; 8] = { +/// 3 = status-busy skip, 4 = jitcounter FIRED (true), 5 = stack_almost_full +/// returned true, 6 = start_retrace_from_guard entered, 7 = start_retrace bailed +/// (source loop evicted: compiled_loops miss), 8 = compile_bridge entered (trace +/// closed → backend request path), 9 = compile_bridge InvalidLoop discard, 10 = +/// compile_bridge retrace_requested return, 11 = compile_bridge arity giveup +/// return (JUMP args != target LABEL args), 12 = start_bridge_tracing entered, +/// 13 = sbt early: descr not FailDescr, 14 = sbt early: no owning jct, 15 = sbt +/// early: no compiled_meta, 16 = sbt early: !can_trace, 17 = sbt early: +/// fail_values too short. +pub static MC_DIAG: [std::sync::atomic::AtomicU64; 18] = { const Z: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - [Z, Z, Z, Z, Z, Z, Z, Z] + [Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z] }; /// Read an `MC_DIAG` tally (saturating). Surfaced via `pyre_jit_mc_diag`. @@ -442,7 +448,11 @@ pub fn mc_diag_bump(i: usize) { #[inline] pub fn stack_almost_full() -> bool { if let Some(f) = STACK_ALMOST_FULL_FN.get() { - f() + let r = f(); + if r { + mc_diag_bump(5); // stack_almost_full returned true + } + r } else { false } diff --git a/majit/majit-metainterp/src/optimizeopt/heap.rs b/majit/majit-metainterp/src/optimizeopt/heap.rs index a40ba96a6a2..cd7323d081d 100644 --- a/majit/majit-metainterp/src/optimizeopt/heap.rs +++ b/majit/majit-metainterp/src/optimizeopt/heap.rs @@ -1931,7 +1931,27 @@ impl OptHeap { // this exact field descr by pointer identity. Gated on // `!compute_bitstrings_has_run()` so the translated interpreter's // bitstring path is unchanged. + // + // STRUCTURAL ADAPTATION (no heap.py analog): upstream every + // cached descr is inside the `compute_bitstrings` universe, so + // the bitcheck is total. Pyre also caches RUNTIME-minted + // descrs (`ei_index` unset even after bitstrings ran — e.g. + // the module-global cell fold's mutable + // `ObjectMutableCell.w_value`). No call's write bitstring can + // ever name such a descr, so a bitcheck miss proves nothing; + // treating it as "not written" let a namespace-store residual + // (analyzed write set, which cannot include the runtime descr) + // keep the getfield cache and re-read a stale global + // (`acc = acc + a; acc = acc + b` at module level dropped the + // first term). A mutable out-of-universe descr is therefore + // conservatively invalidated by every non-elidable call — but + // only once bitstrings have run: before that, a u32::MAX + // `effect_idx` just means the fixture never stamped the slot, so + // the identity fallback below is the correct arbiter. let writes_field = ei.check_write_descr_field(effect_idx) + || (majit_ir::effectinfo::compute_bitstrings_has_run() + && effect_idx == u32::MAX + && !descr.is_always_pure()) || (!majit_ir::effectinfo::compute_bitstrings_has_run() && ei.writes_field_descr_by_identity(&descr)); if writes_field { @@ -1959,7 +1979,14 @@ impl OptHeap { .collect(); for (descr_idx, descr, effect_idx) in array_descrs { let read = ei.check_readonly_descr_array(effect_idx); - let write = ei.check_write_descr_array(effect_idx); + // See the field loop above: a RUNTIME-minted array descr + // (`ei_index` unset) is outside the bitstring universe, so no + // call's write bitstring can name it — conservatively treat + // every non-elidable call as writing it. + let write = ei.check_write_descr_array(effect_idx) + || (majit_ir::effectinfo::compute_bitstrings_has_run() + && effect_idx == u32::MAX + && !descr.is_always_pure()); if !read && !write { continue; } diff --git a/majit/majit-metainterp/src/pyjitpl.rs b/majit/majit-metainterp/src/pyjitpl.rs index 8578603946a..70ea0a6e383 100644 --- a/majit/majit-metainterp/src/pyjitpl.rs +++ b/majit/majit-metainterp/src/pyjitpl.rs @@ -9808,6 +9808,7 @@ impl MetaInterp { snapshot_frame_pcs: SnapshotFramePcs, call_pure_results: majit_ir::VecMap, Value>, ) -> bool { + crate::mc_diag_bump(8); // compile_bridge entered if !self.compiled_loops.contains_key(&green_key) { return false; } @@ -10101,6 +10102,7 @@ impl MetaInterp { // speculative heap access proven ill-typed (now a deferred // `InvalidLoop` signal, not a panic), discards the bridge. Err(inv) => { + crate::mc_diag_bump(9); // compile_bridge InvalidLoop discard if crate::majit_log_enabled() { eprintln!( "[jit] compile_bridge: InvalidLoop(\"{}\") at key={} fail_index={}", @@ -10117,6 +10119,7 @@ impl MetaInterp { // directly; missing-constant recovery from source_trace is // pyre-only and violates bridge pool isolation. if retrace_requested { + crate::mc_diag_bump(10); // compile_bridge retrace_requested return // compile.py:1079: metainterp.retrace_needed(new_trace, info) // Save partial trace + exported state so the next loop-header's // compile_loop → compile_retrace can produce a new specialization. @@ -10198,6 +10201,7 @@ impl MetaInterp { if let Some(target_len) = target_len { let jump_len = jump.getarglist().len(); if target_len != 0 && jump_len != target_len { + crate::mc_diag_bump(11); // compile_bridge arity giveup return if crate::majit_log_enabled() { eprintln!( "[jit] compile_bridge giveup: JUMP args {jump_len} != \ @@ -10429,7 +10433,7 @@ impl MetaInterp { // op-lowering gaps) may be resolved on a differently-shaped // retrace. Record the source guard so `must_compile_with_values` // stops firing for it; the guard then resolves through blackhole - // resume (the always-correct fallback the dormant path uses). + // resume (the always-correct fallback). if matches!(e, majit_backend::BackendError::Unsupported(_)) && self.backend.bridge_decline_is_terminal() { diff --git a/majit/majit-metainterp/src/pyjitpl/dispatch.rs b/majit/majit-metainterp/src/pyjitpl/dispatch.rs index b58b8948adc..ec862d763ac 100644 --- a/majit/majit-metainterp/src/pyjitpl/dispatch.rs +++ b/majit/majit-metainterp/src/pyjitpl/dispatch.rs @@ -616,7 +616,7 @@ fn size_descr_ref_from_bh(descr: &crate::blackhole::BhDescr) -> majit_ir::DescrR *size, *type_id as u32, *type_id, - *vtable, + *vtable as usize, *is_gc_managed, &specs, ); @@ -682,7 +682,7 @@ fn field_descr_ref_from_bh(descr: &crate::blackhole::BhDescr) -> (usize, majit_i p.size, p.type_id as u32, p.type_id, - p.vtable, + p.vtable as usize, p.is_gc_managed, &specs, ); diff --git a/majit/majit-translate/src/codewriter/assembler.rs b/majit/majit-translate/src/codewriter/assembler.rs index 0e20f326910..2499b1f6927 100644 --- a/majit/majit-translate/src/codewriter/assembler.rs +++ b/majit/majit-translate/src/codewriter/assembler.rs @@ -1524,7 +1524,7 @@ impl Assembler { let descr_idx = self.emit_ready_descr(crate::jitcode::BhDescr::Size { size: spec.size, type_id: spec.type_id, - vtable: *vtable as usize, + vtable: *vtable as u64, // `STRUCT._name` identity is left empty for the transient // `bh_new_with_vtable` size descr; the gc_cache hit keys on // `type_id` (`path_hash(owner)`), not this field. @@ -3373,7 +3373,7 @@ fn bh_size_spec_from_descr(sd: &dyn majit_ir::descr::SizeDescr) -> crate::jitcod // which lands on a DIFFERENT cache slot than the analyzer's // path_hash key, polluting cross-path identity. type_id: sd.cache_key(), - vtable: sd.vtable(), + vtable: sd.vtable() as u64, // Round-trip the GC-header flag off the descr so a raw native // struct stays raw through the inverse path (it must not regain // a spurious `GUARD_GC_TYPE`). @@ -4070,7 +4070,7 @@ enum AssemblerDescrKey { size: usize, /// u64 cache-key surrogate matching `BhDescr::Size.type_id`. type_id: u64, - vtable: usize, + vtable: u64, owner: String, all_fielddescrs: Vec, }, diff --git a/majit/majit-translate/src/codewriter/jitcode.rs b/majit/majit-translate/src/codewriter/jitcode.rs index 691b42e7c86..bca15602f15 100644 --- a/majit/majit-translate/src/codewriter/jitcode.rs +++ b/majit/majit-translate/src/codewriter/jitcode.rs @@ -1076,7 +1076,15 @@ pub struct BhSizeSpec { /// :bh_size_spec_from_callcontrol`), so the two routes converge on /// the same `LLType::Struct(u64)` cache key in `gc_cache._cache_size`. pub type_id: u64, - pub vtable: usize, + /// ob_type pointer captured in the PRODUCING process (build script + /// for `opcode_descrs.bin`, live runtime for tracer-minted specs). + /// Declared `u64`, not `usize`: the spec crosses the build→runtime + /// serialization boundary, and a 64-bit host pointer must survive + /// deserialization on a 32-bit (wasm32) runtime instead of failing + /// bincode's width check. Cross-process values are stale under ASLR + /// either way — consumers treat them as opaque identity words and + /// re-resolve real vtables via `type_id` → `gc_cache` publish. + pub vtable: u64, /// True when the struct carries a GC header (`ref - 8` type-id word), /// false for a natively-allocated raw struct registered via /// `register_struct_layout`. Threaded to `SimpleSizeDescr.is_gc_managed` @@ -1226,7 +1234,10 @@ pub enum BhDescr { /// u64 `path_hash(module_path::Struct)` — see /// `BhSizeSpec.type_id` doc for full identity rationale. type_id: u64, - vtable: usize, + /// See `BhSizeSpec.vtable`: producer-process ob_type pointer, + /// `u64` for wire-width stability across the build→runtime + /// (and 64→32-bit) serialization boundary. + vtable: u64, /// RPython `STRUCT._name` identity (empty when the size descr /// is built transiently for `bh_new` / `bh_new_with_vtable` /// dispatch and the struct identity is already encoded in the @@ -1323,7 +1334,7 @@ impl BhDescr { pub fn get_vtable(&self) -> usize { match self { - BhDescr::Size { vtable, .. } => *vtable, + BhDescr::Size { vtable, .. } => *vtable as usize, _ => 0, } } diff --git a/pyre/check.py b/pyre/check.py index 3d678962337..bd2ce985ad8 100644 --- a/pyre/check.py +++ b/pyre/check.py @@ -5,8 +5,10 @@ """ import argparse +import math import os import shutil +import struct import subprocess import sys import time @@ -343,10 +345,96 @@ def default_binary(backend): return f"./target/release/{name}{EXE}" +# Relative tolerance for wasm float outputs ONLY (see `wasm_outputs_match`). +WASM_FLOAT_RTOL = 1e-9 + + +def wasm_outputs_match(output, expected): + """Bench-output comparison for the wasm backend, allowing a bounded + float divergence — used ONLY for benches explicitly opted in with + run_bench(..., wasm_float_tol=True). No blanket wasm allowance exists; + today only nbody is marked, and every other wasm bench stays byte-exact. + + Root cause (measured, not guessed): the wasm guest and native pyre run + the SAME libm source, but libm's transcendentals (`pow` is the FreeBSD + `e_pow.c` hi/lo split) bottom out on ARCH-SPECIFIC building blocks + (`libm/src/math/arch/aarch64.rs`). On aarch64 those land on hardware-tuned + ops that match the platform macOS libm — so native pyre matches + CPython/PyPy bit-for-bit — while wasm32 gets the generic software + fallbacks. That is a ~0.5-ULP-per-op gap that accumulates (nbody: 5M pow + calls -> ~1490 ULP, 3e-13 relative, in the printed energy). It is an + unfixable target-ISA codegen gap, not a miscompile: an FMA-fusion + hypothesis was tested and refuted (fp-contract=fast on the wasm build is + a no-op — wasm32 scalar has no FMA instruction — and libm's pow uses no + mul_add), forcing native unfused would break native parity with the + platform reference, and host-libm callbacks were rejected (they make wasm + output vary by host machine, which we do not want). + + So float tokens are compared with a relative tolerance `WASM_FLOAT_RTOL` + (1e-9 — ~4 orders looser than the observed 3e-13 drift, ~3 orders + TIGHTER than the smallest real value bug seen, 5.7e-6). Every non-float + token (ints, strings, non-finite floats) still must match byte-for-byte, + so an int off-by-one can never slip through.""" + if output == expected: + return True + out_lines, exp_lines = output.splitlines(), expected.splitlines() + if len(out_lines) != len(exp_lines): + return False + for out_line, exp_line in zip(out_lines, exp_lines): + if out_line == exp_line: + continue + out_toks, exp_toks = out_line.split(), exp_line.split() + if len(out_toks) != len(exp_toks): + return False + for out_tok, exp_tok in zip(out_toks, exp_toks): + if out_tok == exp_tok: + continue + # Only finite float-shaped tokens (decimal point or exponent) + # get tolerance; ints/strings/nan/inf must match byte-for-byte + # above, so an int off-by-one can never slip through. + def _floaty(tok): + return "." in tok or "e" in tok or "E" in tok + if not (_floaty(out_tok) and _floaty(exp_tok)): + return False + try: + a, b = float(out_tok), float(exp_tok) + except ValueError: + return False + if not (math.isfinite(a) and math.isfinite(b)): + return False + if abs(a - b) > WASM_FLOAT_RTOL * max(abs(a), abs(b)): + return False + return True + + # Backends rendered in fixed-column displays, in order. Any enabled backend not # listed here still runs and is counted; it just falls outside the fixed columns. ALL_BACKENDS = ("dynasm", "cranelift", "wasm") + + +def _wasm_target_installed(): + """Whether the wasm backend can be built here. + + The only extra prerequisite over the native backends is the + `wasm32-unknown-unknown` rustup target (the wasmtime runtime is embedded + in `pyre-wasm-runner`, not an external tool). If it is missing, the wasm + build would `rustup target add`-fail, so wasm stays out of the default set. + """ + try: + proc = subprocess.run( + ["rustup", "target", "list", "--installed"], + capture_output=True, text=True, timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return False + return proc.returncode == 0 and "wasm32-unknown-unknown" in proc.stdout.split() + + +# wasm joins the defaults only where its target is installed, so a plain +# `check.py` on an unconfigured machine still runs just the native backends. DEFAULT_BACKENDS = ("dynasm", "cranelift") +if _wasm_target_installed(): + DEFAULT_BACKENDS = (*DEFAULT_BACKENDS, "wasm") # ── Check runner ───────────────────────────────────────────────────── @@ -663,6 +751,7 @@ def warmup(self, script): def _run_backend_bench( self, backend, name, script, timeout, vs_cpython, vs_pypy, t_cpython, t_pypy, pypy_output, + wasm_float_tol=False, ): pyre_bin = self._pyre(backend) effective_timeout = scaled_timeout(timeout, self._timeout_scale(backend)) @@ -691,7 +780,17 @@ def _run_backend_bench( self._append_comparison(backend, name, t_cpython, t_pypy, "FAIL") return - if output != pypy_output: + # Every backend requires byte-identical output. The sole exception is a + # bench explicitly marked wasm_float_tol=True AND run on the wasm + # backend: wasm32's scalar ISA cannot reproduce the platform libm's + # arch-tuned pow, so its transcendental-heavy output drifts by a bounded + # amount (see `wasm_outputs_match`). This is opt-in per bench, never a + # blanket wasm allowance — every other wasm bench stays byte-exact. + if backend == "wasm" and wasm_float_tol: + matched = wasm_outputs_match(output, pypy_output) + else: + matched = output == pypy_output + if not matched: exp = pypy_output[:60] act = output[:60] self._record(backend, False, name, "wrong output") @@ -751,7 +850,7 @@ def run_bench( self, name, script, timeout, dynasm_vs_cpython=None, dynasm_vs_pypy=None, cranelift_vs_cpython=None, cranelift_vs_pypy=None, - skip_backends=(), + skip_backends=(), wasm_float_tol=False, ): need_cpython = False if ( @@ -819,6 +918,7 @@ def run_bench( self._run_backend_bench( backend, name, script, timeout, vs_cpython, vs_pypy, t_cpython, t_pypy, pypy_output, + wasm_float_tol=wasm_float_tol, ) # ── synthetic parity suite ── @@ -1148,7 +1248,7 @@ def main(): chk.run_bench("nested_loop", f"{B}/nested_loop.py", 5, None, 2, None, 3) chk.run_bench("raise_catch", f"{B}/raise_catch_loop.py", 5, None, 1.5, None, 2.5) chk.run_bench("spectral_norm", f"{B}/spectral_norm.py", 5, 2, 7, 2, 7) - chk.run_bench("nbody", f"{B}/nbody.py", 10, 3, None, 3, None) + chk.run_bench("nbody", f"{B}/nbody.py", 10, 3, None, 3, None, wasm_float_tol=True) chk.run_bench("fannkuch", f"{B}/fannkuch.py", 30, 1, 5, 2, None) if not args.no_synthetic: diff --git a/pyre/pyre-interpreter/src/builtins.rs b/pyre/pyre-interpreter/src/builtins.rs index 7a84471f4e2..6b59f53c9f9 100644 --- a/pyre/pyre-interpreter/src/builtins.rs +++ b/pyre/pyre-interpreter/src/builtins.rs @@ -4946,10 +4946,10 @@ pub(crate) fn builtin_float(args: &[PyObjectRef]) -> Result'"` — since setattr/delattr get their check there. -/// getattr/hasattr use their own `bltinmodule.c` message inline. +/// (accept `str` and any `str` subclass via `isinstance_w`), raising +/// `"attribute name must be string, not ''"`. getattr/hasattr/ +/// setattr/delattr all route through it, matching `operation.py` (and the +/// unified 3.12+ message). fn checkattrname(w_name: PyObjectRef) -> Result<(), crate::PyError> { if !unsafe { crate::baseobjspace::isinstance_str_w(w_name) } { let name_type = unsafe { (*(*w_name).ob_type).name }; @@ -4960,9 +4960,9 @@ fn checkattrname(w_name: PyObjectRef) -> Result<(), crate::PyError> { Ok(()) } -/// `hasattr(obj, name)` → bool. `builtin_hasattr` rejects a non-str name -/// with `"hasattr(): attribute name must be string"`, then (unlike Py2) -/// only an `AttributeError` yields `False`; any other error propagates. +/// `operation.py:65-74 hasattr(obj, name)` → bool: `checkattrname`, then +/// (unlike Py2) only an `AttributeError` yields `False`; any other error +/// propagates. fn builtin_hasattr(args: &[PyObjectRef]) -> Result { if args.len() != 2 { return Err(crate::PyError::type_error(format!( @@ -4971,11 +4971,7 @@ fn builtin_hasattr(args: &[PyObjectRef]) -> Result ))); } let obj = args[0]; - if !unsafe { crate::baseobjspace::isinstance_str_w(args[1]) } { - return Err(crate::PyError::type_error( - "hasattr(): attribute name must be string", - )); - } + checkattrname(args[1])?; match crate::baseobjspace::getattr(obj, args[1]) { Ok(_) => Ok(w_bool_from(true)), Err(e) if e.kind == crate::PyErrorKind::AttributeError => Ok(w_bool_from(false)), @@ -4999,11 +4995,7 @@ fn builtin_getattr(args: &[PyObjectRef]) -> Result ))); } let obj = args[0]; - if !unsafe { crate::baseobjspace::isinstance_str_w(args[1]) } { - return Err(crate::PyError::type_error( - "getattr(): attribute name must be string", - )); - } + checkattrname(args[1])?; // operation.py:58-64: the default replaces the error ONLY when a default // was supplied AND the error is an AttributeError; other errors (and the // no-default case) propagate. diff --git a/pyre/pyre-jit-trace/src/descr.rs b/pyre/pyre-jit-trace/src/descr.rs index 96f06e64f79..d3bf8133645 100644 --- a/pyre/pyre-jit-trace/src/descr.rs +++ b/pyre/pyre-jit-trace/src/descr.rs @@ -2887,7 +2887,7 @@ fn simple_descr_group_from_bh_size( spec.size, spec.type_id as u32, spec.type_id, - spec.vtable, + spec.vtable as usize, spec.is_gc_managed, &field_specs, ) @@ -3623,7 +3623,7 @@ pub fn make_descr_from_bh(bh: &majit_translate::jitcode::BhDescr) -> DescrRef { // TODO: `make_size_descr_with_type_and_vtable` // takes the u32 gc tid; `*type_id` is the u64 cache key. // Truncate `as u32` until gc_cache routing. - make_size_descr_with_type_and_vtable(*size, *type_id as u32, *vtable) + make_size_descr_with_type_and_vtable(*size, *type_id as u32, *vtable as usize) } else { let spec = majit_translate::jitcode::BhSizeSpec { size: *size, diff --git a/pyre/pyre-jit-trace/src/helpers.rs b/pyre/pyre-jit-trace/src/helpers.rs index dc3a5e728b0..9df925cadf7 100644 --- a/pyre/pyre-jit-trace/src/helpers.rs +++ b/pyre/pyre-jit-trace/src/helpers.rs @@ -228,13 +228,29 @@ pub fn emit_trace_call_void(ctx: &mut TraceCtx, helper: *const (), args: &[OpRef ctx.call_void(helper, args); } -pub fn emit_trace_call_void_typed( +/// Record a void residual whose hand-written `extern "C"` helper returns a +/// dummy machine word (`-> i64`, value ignored) — the convention of this +/// module's i64-ABI wrappers ([`jit_store_name_to_namespace`], +/// [`jit_list_append`]). The word-ABI descr lets a signature-exact backend +/// lowering (wasm direct `call_indirect`) call the helper in-module. A +/// helper that genuinely returns `()` must use `ctx.call_void_typed` +/// instead. +/// +/// `effect_info`: these helpers write the heap (namespace cells, list +/// storage), so the caller must supply the effect — normally +/// `EffectInfo::MOST_GENERAL` (`graphanalyze.py:60 +/// analyze_external_call` top for an unanalyzed external writer). The +/// opcode-default empty write set would let optheap CSE a getfield +/// across the call: `acc = acc + a; acc = acc + b` at module level then +/// reuses the pre-store cell value and drops the first term. +pub fn emit_trace_call_void_word_abi( ctx: &mut TraceCtx, helper: *const (), args: &[OpRef], arg_types: &[Type], + effect_info: majit_ir::EffectInfo, ) { - ctx.call_void_typed(helper, args, arg_types); + ctx.call_void_typed_word_abi(helper, args, arg_types, effect_info); } pub fn emit_trace_call_may_force_ref_typed( @@ -371,11 +387,18 @@ pub fn emit_trace_store_name_to_namespace( value: OpRef, ) { let [name_ptr, name_len] = trace_name_args(ctx, name); - emit_trace_call_void_typed( + // The helper runs `w_dict_setitem_str` → ModuleDictStrategy + // `write_cell` — an in-place `ObjectMutableCell.w_value` write with + // no version bump, exactly what `load_name_value`'s cell fast path + // reads back as `GetfieldGcR(cell)`. MOST_GENERAL makes the + // optimizer drop that field cache so the next LOAD re-reads the + // cell instead of reusing the pre-store value. + emit_trace_call_void_word_abi( ctx, jit_store_name_to_namespace as *const (), &[namespace, name_ptr, name_len, value], &[Type::Ref, Type::Int, Type::Int, Type::Ref], + majit_ir::EffectInfo::MOST_GENERAL, ); } @@ -622,11 +645,15 @@ pub trait TraceHelperAccess { fn trace_list_append(&mut self, list: OpRef, value: OpRef) -> Result<(), PyError> { self.with_trace_ctx(|ctx| { - emit_trace_call_void_typed( + // Writes the list's strategy storage (may also realloc it); + // see `emit_trace_call_void_word_abi` — an unanalyzed + // external writer records MOST_GENERAL. + emit_trace_call_void_word_abi( ctx, jit_list_append as *const (), &[list, value], &[Type::Ref, Type::Ref], + majit_ir::EffectInfo::MOST_GENERAL, ); }); self.trace_record_no_exception_guard(); diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch.rs index 86f09cf4fc4..fbb7fd2712c 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch.rs @@ -12566,9 +12566,16 @@ fn dispatch_residual_call_iRd_kind( && ctx.is_full_body_walk && dst_bank == 'v' && ei.pyre_helper == majit_ir::PyreHelperKind::StoreSubscr - && try_walker_specialize_store_subscr(ctx, op.pc, &r_args)?.is_some() { - return Ok((DispatchOutcome::Continue, op.next_pc)); + if try_walker_specialize_store_subscr(ctx, op.pc, &r_args)?.is_some() { + return Ok((DispatchOutcome::Continue, op.next_pc)); + } else if ctx.trace_ctx.is_bridge_trace && fbw_debug_abort_enabled() { + eprintln!( + "[fbw-store-fallthrough] bridge STORE_SUBSCR fell to GENERIC residual at pc={} \ + (specialization declined — unjournaled concrete store)", + op.pc + ); + } } // #195 / #73: virtualize an arity-2 plain-int BUILD_TUPLE diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 449164351e7..f78e7fa4f8d 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -5684,7 +5684,7 @@ fn bh_size_descr_from_size_descr( // descr.py type_id is the dense GC tid for alloc_nursery_typed, // not the cache_key structural identity type_id: size_descr.type_id() as u64, - vtable, + vtable: vtable as u64, owner: String::new(), all_fielddescrs: majit_translate::jitcode::bh_field_specs_from_size_descr(size_descr), // Round-trip the GC-header flag off the descr. diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 07410031342..6b1de98c09e 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -1314,7 +1314,13 @@ fn run_perfn_walk( // replay applies) is present: drop the capture so the portal degrades // to `ContinueRunningNormally`. This shares its predicate with the // store-journal commit below so the two decisions never disagree. + // + // Bridge walks are excluded: only the loop-free function portal consumes + // the finish stash without replaying. A bridge `Terminate` walk's caller + // resumes the region through the blackhole, so keeping the stores here + // would double-apply them (once eagerly, once in the blackhole replay). let terminate_no_replay = crate::jitcode_dispatch::fbw_no_replay_exit_enabled() + && !is_bridge_trace && matches!( &walk_result, Ok((crate::jitcode_dispatch::DispatchOutcome::Terminate, _)) @@ -1333,6 +1339,30 @@ fn run_perfn_walk( // `terminate_no_replay` exit also keeps the stores: the portal returns // the walk's result without replaying, exactly like the loop-flush // commit. + if is_bridge_trace && crate::jitcode_dispatch::fbw_debug_abort_enabled() { + let outcome_kind = match &walk_result { + Ok((crate::jitcode_dispatch::DispatchOutcome::Continue, _)) => "Continue", + Ok((crate::jitcode_dispatch::DispatchOutcome::Terminate, _)) => "Terminate", + Ok((crate::jitcode_dispatch::DispatchOutcome::SubReturn { .. }, _)) => "SubReturn", + Ok((crate::jitcode_dispatch::DispatchOutcome::SubRaise { .. }, _)) => "SubRaise", + Ok((crate::jitcode_dispatch::DispatchOutcome::SwitchToBlackhole { .. }, _)) => { + "SwitchToBlackhole" + } + Ok((crate::jitcode_dispatch::DispatchOutcome::CloseLoop { .. }, _)) => "CloseLoop", + Ok((crate::jitcode_dispatch::DispatchOutcome::CompileTracePending { .. }, _)) => { + "CompileTracePending" + } + Ok((_, _)) => "OtherOk", + Err(_) => "Err", + }; + eprintln!( + "[fbw-bridge-epilogue] committed={} store_journal_len={} unjournaled={} outcome={}", + WALK_END_FLUSH_COMMITTED.with(|c| c.get()), + crate::jitcode_dispatch::fbw_store_journal_len(), + crate::jitcode_dispatch::fbw_has_unjournaled_effect(), + outcome_kind, + ); + } if WALK_END_FLUSH_COMMITTED.with(|c| c.get()) || terminate_no_replay { crate::jitcode_dispatch::fbw_store_journal_commit(); } else { diff --git a/pyre/pyre-jit/src/call_jit.rs b/pyre/pyre-jit/src/call_jit.rs index a0018effc98..bc93b5318df 100644 --- a/pyre/pyre-jit/src/call_jit.rs +++ b/pyre/pyre-jit/src/call_jit.rs @@ -2125,14 +2125,13 @@ fn bridge_source_identity_from_descr( /// On failure (trace abort, start failure), returns false so the caller /// falls through to resume_in_blackhole (RPython pyjitpl.py:2906-2907 /// SwitchToBlackhole → run_blackhole_interp_to_cancel_tracing). -/// Runtime toggle for the otherwise-dormant wasm bridge tracer (default off). -/// Set by the runner via the `pyre_jit_set_enable_bridges` export so chaining -/// can be enabled/measured without rebuilding the guest module. Kept off by -/// default — chaining does not yet resolve the bench timeouts and re-enabling -/// it surfaces a loop-closing bridge livelock on some traces (fannkuch). +/// Runtime toggle for the wasm bridge tracer (default ON). The runner's +/// `pyre_jit_set_enable_bridges` export flips it (`PYRE_WASM_ENABLE_BRIDGES=0` +/// disables) so chaining can be A/B-measured without rebuilding the guest +/// module. #[cfg(target_arch = "wasm32")] pub static WASM_BRIDGES_ENABLED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); + std::sync::atomic::AtomicBool::new(true); /// Set the wasm bridge-tracer enable flag (see `WASM_BRIDGES_ENABLED`). #[cfg(target_arch = "wasm32")] @@ -2184,21 +2183,9 @@ pub fn trace_and_compile_from_bridge( return false; }; - // The wasm bridge tracer stays dormant. Inter-trace chaining (this path's - // loop-closing `return_call_indirect` bridges) is implemented and runs - // correctly in-module, but it does not resolve the wasm bench timeouts: the - // dominant per-iteration cost is the residual-call host crossings the trace - // already performs (~33 `jit_call` round-trips per bool_arithmetic iteration - // — every int/float op and box is a residual call routed guest→host→guest - // through the `jit_call` trampoline, ~15µs each). Chaining removes only the - // per-guard-exit round-trips, not these per-op ones, so an allocating hot - // loop stays ~50x slower than native (which inlines the arithmetic). The fix - // is a separate wasm-codegen epic: emit residual calls as direct guest→guest - // `call_indirect` (no host trampoline) or inline int/float arithmetic in the - // trace. Keeping the bridge tracer dormant avoids the chained loop's residual - // old-gen growth (the in-loop GC reclamation story, gated on a faithful - // incremental-major pacing fix that is not yet safe) until that lands. - // Removing this re-enables the (otherwise complete and verified) chaining. + // Wasm bridge tracer kill switch (`WASM_BRIDGES_ENABLED`, default ON): + // declining here drops every guard failure to blackhole-from-guard, the + // chaining-free fallback, for A/B measurement. #[cfg(target_arch = "wasm32")] if !WASM_BRIDGES_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { let _ = ( @@ -2271,36 +2258,6 @@ pub fn trace_and_compile_from_bridge( "[jit][bridge-trace] start key={} trace={} fail={} resume_pc={}", green_key, trace_id, fail_index, resume_pc ); - if trace_id == 2 && fail_index == 2 && resume_pc == 153 { - let debug_values: Vec = raw_values - .iter() - .zip(exit_layout.exit_types.iter()) - .enumerate() - .map(|(idx, (&raw, &tp))| match tp { - majit_ir::Type::Ref => { - let obj = raw as pyre_object::PyObjectRef; - let detail = unsafe { - if obj.is_null() { - "null".to_string() - } else if pyre_object::is_float(obj) { - format!("float({})", pyre_object::w_float_get_value(obj)) - } else if pyre_object::is_int(obj) { - format!("int({})", pyre_object::w_int_get_value(obj)) - } else if pyre_object::is_list(obj) { - "list".to_string() - } else { - format!("ref({:#x})", obj as usize) - } - }; - format!("#{idx}:Ref {detail}") - } - majit_ir::Type::Int => format!("#{idx}:Int {}", raw), - majit_ir::Type::Float => format!("#{idx}:Float {}", f64::from_bits(raw as u64)), - majit_ir::Type::Void => format!("#{idx}:Void"), - }) - .collect(); - eprintln!("[jit][bridge-raw] {}", debug_values.join(", ")); - } } // compile.py:714: start_retrace_from_guard + set bridge_info. @@ -2317,7 +2274,6 @@ pub fn trace_and_compile_from_bridge( } return false; } - // RPython pyjitpl.py:3101 _prepare_exception_resumption + // pyjitpl.py:3132 prepare_resume_from_failure parity: // For exception guard bridges (GUARD_EXCEPTION / GUARD_NO_EXCEPTION), diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index 60150f9f60e..e8ee9971f5b 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -5496,7 +5496,7 @@ fn allocate_with_vtable(descr: &dyn majit_ir::SizeDescr) -> usize { size, // `descr.py:108-118` cache identity via `SizeDescr.cache_key()`. type_id: descr.cache_key(), - vtable, + vtable: vtable as u64, owner: String::new(), all_fielddescrs: majit_translate::jitcode::bh_field_specs_from_size_descr(descr), is_gc_managed: descr.is_gc_managed(), @@ -7641,7 +7641,7 @@ impl majit_metainterp::resume::BlackholeAllocator for PyreBlackholeAllocator { size: descr_size, // Note: u32 gc tid widened to u64 cache key slot. type_id: descr_index as u64, - vtable, + vtable: vtable as u64, owner: String::new(), all_fielddescrs: majit_translate::jitcode::bh_field_specs_from_size_descr(sd), is_gc_managed: sd.is_gc_managed(), diff --git a/pyre/pyre-jit/src/lib.rs b/pyre/pyre-jit/src/lib.rs index 08f0fc44747..d5f3e9f1344 100644 --- a/pyre/pyre-jit/src/lib.rs +++ b/pyre/pyre-jit/src/lib.rs @@ -67,6 +67,13 @@ pub fn wasm_gc_heap_stats() -> (usize, usize) { majit_backend_wasm::active_gc_heap_stats() } +/// Diagnostic only: `(minor_collections, major_collections)` of the wasm +/// backend's GC on this thread. Companion to [`wasm_gc_heap_stats`]. +#[cfg(target_arch = "wasm32")] +pub fn wasm_gc_collection_counts() -> (usize, usize) { + majit_backend_wasm::active_gc_collection_counts() +} + #[cfg(test)] mod tests { use super::*; diff --git a/pyre/pyre-wasm-runner/src/main.rs b/pyre/pyre-wasm-runner/src/main.rs index a9f1c576b1b..12dfaefdeb7 100644 --- a/pyre/pyre-wasm-runner/src/main.rs +++ b/pyre/pyre-wasm-runner/src/main.rs @@ -90,6 +90,14 @@ struct Host { /// already hold the `Caller`, and the runner is single-threaded. jit_compile_count: u64, jit_execute_count: u64, + /// Diagnostic: per-op residual-call host crossings (`env.jit_call` + /// trampoline invocations). Compare against `jit_execute_count` to test + /// whether per-op crossings or per-guard-exit crossings dominate. + jit_call_count: u64, + /// Diagnostic (PYRE_WASM_EXEC_TRACE=1): histogram of (trace func_id, + /// guard-exit fail_index) over every host round-trip, so we can see which + /// guard keeps returning to the host instead of chaining in-module. + exec_hist: std::collections::BTreeMap<(u32, u32), u64>, } fn main() { @@ -236,22 +244,33 @@ fn run(module_path: &PathBuf, source: &str) -> Result { let run_python = instance.get_typed_func::<(u32, u32), u64>(&mut store, "pyre_run_python")?; let dealloc = instance.get_typed_func::<(u32, u32), ()>(&mut store, "pyre_dealloc")?; - // Enable the otherwise-dormant wasm bridge tracer (inter-trace chaining) when - // PYRE_WASM_ENABLE_BRIDGES is set, so chaining can be measured without - // rebuilding the guest. No-op if the export is absent (older modules). - if std::env::var_os("PYRE_WASM_ENABLE_BRIDGES").is_some() { + // Wasm bridge tracer (inter-trace chaining) is ON by default in the guest; + // PYRE_WASM_ENABLE_BRIDGES=0 disables it (any other value re-enables) for + // A/B measurement. No-op if the export is absent (older modules). + if let Ok(v) = std::env::var("PYRE_WASM_ENABLE_BRIDGES") { if let Ok(f) = instance.get_typed_func::(&mut store, "pyre_jit_set_enable_bridges") { - f.call(&mut store, 1)?; + f.call(&mut store, u32::from(v != "0"))?; } } - // Enable the self-recursive CALL_ASSEMBLER guest→guest `call_indirect` arm - // (`PYRE_WASM_CA`). The guest has no environment, so the flag is plumbed - // through this export. No-op if the export is absent (older modules). - if std::env::var_os("PYRE_WASM_CA").is_some() { + // Inline nursery-bump allocation fast path is ON by default in the guest; + // PYRE_WASM_INLINE_ALLOC=0 disables it (any other value re-enables) for + // A/B measurement. No-op if the export is absent (older modules). + if let Ok(v) = std::env::var("PYRE_WASM_INLINE_ALLOC") { + if let Ok(f) = instance.get_typed_func::(&mut store, "pyre_jit_set_inline_alloc") { + f.call(&mut store, u32::from(v != "0"))?; + } + } + + // Self-recursive CALL_ASSEMBLER guest→guest `call_indirect` arm is ON by + // default in the guest; PYRE_WASM_CA=0 disables it (any other value + // re-enables) for A/B measurement. The guest has no environment, so the + // flag is plumbed through this export. No-op if the export is absent + // (older modules). + if let Ok(v) = std::env::var("PYRE_WASM_CA") { if let Ok(f) = instance.get_typed_func::(&mut store, "pyre_jit_set_wasm_ca") { - f.call(&mut store, 1)?; + f.call(&mut store, u32::from(v != "0"))?; } } @@ -289,6 +308,14 @@ fn run(module_path: &PathBuf, source: &str) -> Result { .get_typed_func::<(), u64>(&mut store, "pyre_gc_nursery_bytes") .and_then(|f| f.call(&mut store, ())) .unwrap_or(0); + let gc_minors = instance + .get_typed_func::<(), u64>(&mut store, "pyre_gc_minor_collections") + .and_then(|f| f.call(&mut store, ())) + .unwrap_or(0); + let gc_majors = instance + .get_typed_func::<(), u64>(&mut store, "pyre_gc_major_collections") + .and_then(|f| f.call(&mut store, ())) + .unwrap_or(0); // `heap-prof` builds only: net-live guest-heap bytes/count. Distinguishes // a true not-freed leak (live grows with executes) from fragmentation. let heap_live_bytes = instance @@ -327,11 +354,13 @@ fn run(module_path: &PathBuf, source: &str) -> Result { "loopclosing", "src_preamble", "ml_descr_none", - "ml_nonlast", + "ml_unsafe_label", "ml_arity_mismatch", "decl_noadvance", "ca_cell_set", "ca_cells_zero", + "decl_ca_chain", + "reserved15", ]; let mut parts = Vec::new(); for (i, lbl) in labels.iter().enumerate() { @@ -348,9 +377,19 @@ fn run(module_path: &PathBuf, source: &str) -> Result { "descr0_skip", "busy_skip", "FIRED", - "reserved", + "stack_full", "retrace_entered", "retrace_bailed", + "cb_entered", + "cb_invalidloop", + "cb_retrace_req", + "cb_arity_giveup", + "sbt_entered", + "sbt_not_faildescr", + "sbt_no_jct", + "sbt_no_meta", + "sbt_cant_trace", + "sbt_short_vals", ]; let mut parts = Vec::new(); for (i, lbl) in labels.iter().enumerate() { @@ -363,16 +402,28 @@ fn run(module_path: &PathBuf, source: &str) -> Result { } let host = store.data(); eprintln!( - "[jit-stats] compiles={} executes={} linear_mem={} gc_oldgen={} gc_nursery={} \ - heap_live_bytes={} heap_live_count={}", + "[jit-stats] compiles={} executes={} jit_calls={} linear_mem={} gc_oldgen={} gc_nursery={} \ + gc_minors={} gc_majors={} heap_live_bytes={} heap_live_count={}", host.jit_compile_count, host.jit_execute_count, + host.jit_call_count, lin_mem, gc_oldgen, gc_nursery, + gc_minors, + gc_majors, heap_live_bytes, heap_live_count, ); + if !host.exec_hist.is_empty() { + let mut v: Vec<_> = host.exec_hist.iter().collect(); + v.sort_by(|a, b| b.1.cmp(a.1)); + for ((func_id, fail_index), n) in v.into_iter().take(8) { + eprintln!( + "[jit-stats] exec_hist func_id={func_id} fail_index={fail_index} roundtrips={n}" + ); + } + } } let packed = match run_result { Ok(p) => p, @@ -756,14 +807,27 @@ fn jit_execute(caller: &mut Caller<'_, Host>, func_id: u32, frame_ptr: u32) -> R }; let mut results = [Val::I32(0)]; trace.call(&mut *caller, &[Val::I32(frame_ptr as i32)], &mut results)?; - Ok(match results[0] { + let ret = match results[0] { Val::I32(x) => x as u32, _ => 0, - }) + }; + // Diagnostic: record which (trace, guard-exit fail_index) round-tripped. + if std::env::var_os("PYRE_WASM_EXEC_TRACE").is_some() { + if let Some(mem) = caller.data().memory { + let fail_index = read_u32(&mem, &*caller, frame_ptr as usize); + *caller + .data_mut() + .exec_hist + .entry((func_id, fail_index)) + .or_insert(0) += 1; + } + } + Ok(ret) } /// Dispatch a residual call requested by a running trace. fn jit_call_trampoline(caller: &mut Caller<'_, Host>, frame_ptr: u32) -> Result<()> { + caller.data_mut().jit_call_count += 1; let memory = caller.data().memory.context("memory")?; let table = caller.data().table.context("table")?; let frame = frame_ptr as usize; diff --git a/pyre/pyre-wasm/src/lib.rs b/pyre/pyre-wasm/src/lib.rs index 57f00ff17cb..d575ee41626 100644 --- a/pyre/pyre-wasm/src/lib.rs +++ b/pyre/pyre-wasm/src/lib.rs @@ -317,13 +317,17 @@ pub extern "C" fn pyre_jit_mc_diag(i: u32) -> u64 { majit_metainterp::mc_diag(i as usize) } -/// Enable the otherwise-dormant wasm bridge tracer (inter-trace chaining) at -/// runtime, set by the host runner from an env flag. Exported (not an import) -/// for the same function-index-stability reason as the diag readers. +/// Toggle the wasm bridge tracer (inter-trace chaining, default ON) at +/// runtime, set by the host runner from `PYRE_WASM_ENABLE_BRIDGES`. Exported +/// (not an import) for the same function-index-stability reason as the diag +/// readers. #[cfg(all(target_arch = "wasm32", feature = "wasm-host"))] #[unsafe(no_mangle)] pub extern "C" fn pyre_jit_set_enable_bridges(enabled: u32) { pyre_jit::call_jit::set_wasm_bridges_enabled(enabled != 0); + // Backend-side mirror: `execute_token` sizes every host frame's Ref-home + // region for cross-trace chaining when this is on. + majit_backend_wasm::set_wasm_bridges_enabled(enabled != 0); } /// Enable the self-recursive CALL_ASSEMBLER guest→guest `call_indirect` arm @@ -337,6 +341,15 @@ pub extern "C" fn pyre_jit_set_wasm_ca(enabled: u32) { majit_backend_wasm::set_wasm_ca_enabled(enabled != 0); } +/// Toggle the inline nursery-bump allocation fast path (default ON; +/// `PYRE_WASM_INLINE_ALLOC=0` kill switch). Same plumbing rationale as +/// `pyre_jit_set_wasm_ca`. +#[cfg(all(target_arch = "wasm32", feature = "wasm-host"))] +#[unsafe(no_mangle)] +pub extern "C" fn pyre_jit_set_inline_alloc(enabled: u32) { + majit_backend_wasm::set_wasm_inline_alloc_enabled(enabled != 0); +} + #[cfg(any(feature = "web", feature = "wasm-host"))] static PANIC_HOOK: Once = Once::new(); @@ -376,6 +389,13 @@ fn run_python_impl(source: &str) -> String { install_panic_hook(); #[cfg(all(target_arch = "wasm32", feature = "wasm-host"))] residual_host::install(); + // Eagerly install pyre-jit's hooks (pyrex real_main does the same at + // boot): the dict `eq_w` / `hash_w` / `hash_str` / + // `compares_by_identity` trampolines must be live before + // `install_builtin_modules` / `import` builds the first str- or + // object-keyed dict, not only after the first JIT-traced bytecode + // (`dict_eq_hook::missing_hash_hook` fails fast otherwise). + pyre_jit::eval::init_jit_hooks(); pyre_interpreter::importing::install_builtin_modules(); // Give the import machinery a source of module bytes. The browser has no // filesystem, so the web build serves the embedded stdlib closure from an @@ -529,6 +549,18 @@ mod host_abi { pyre_jit::wasm_gc_heap_stats().1 as u64 } + /// Diagnostic: minor collections run so far, or 0 if no GC is installed. + #[unsafe(no_mangle)] + pub extern "C" fn pyre_gc_minor_collections() -> u64 { + pyre_jit::wasm_gc_collection_counts().0 as u64 + } + + /// Diagnostic: major collections run so far, or 0 if no GC is installed. + #[unsafe(no_mangle)] + pub extern "C" fn pyre_gc_major_collections() -> u64 { + pyre_jit::wasm_gc_collection_counts().1 as u64 + } + /// Run the UTF-8 Python source at `ptr[..len]`. Returns a packed /// `(result_ptr << 32) | result_len`; the result is a UTF-8 byte buffer /// the host must free with `pyre_dealloc`.