diff --git a/majit/majit-backend-wasm/src/codegen.rs b/majit/majit-backend-wasm/src/codegen.rs index f3506bee71b..6af734bd3bb 100644 --- a/majit/majit-backend-wasm/src/codegen.rs +++ b/majit/majit-backend-wasm/src/codegen.rs @@ -2050,17 +2050,6 @@ fn build_function( // Def / last-use positions for the post-collection Ref reload filter. let liveness = HomeLiveness::collect(inputargs, ops); - // A Label-less trace with bridge dispatch — a `PYRE_WASM_CA` recursion - // loop, or (chaining on) a bridge whose own guards chain nested - // sub-bridges: there is no `loop`, but its guard/Finish exits still need - // to `br` to the function epilogue so the epilogue's cell dispatch can - // chain a failing guard in-module (instead of each guard early-returning - // to the host). Wrap the body in one exit `block` and route exits through - // it, exactly as a loop does. A loop-closing bridge's terminal external - // JUMP is unaffected — `return_call_indirect` leaves the function from - // inside the block. - let straightline_dispatch = !has_loop && bridge_dispatch; - // Resume-at-LABEL: a peeled loop wraps its preamble in a dispatch so a // loop-closing bridge can re-enter AT any LABEL — key = label ordinal + 1 // — skipping the code before it, in-module instead of round-tripping @@ -2085,15 +2074,20 @@ fn build_function( .take(num_labels) .map(|op| op.getarglist().iter().map(|a| a.to_opref()).collect()) .collect(); + + // x86/assembler.py:835-846 `write_pending_failure_recoveries` places the + // guard recovery stubs after the hot trace. Wasm uses one universal hot + // exit block and records only an exit ordinal at each guard site; the + // fail-argument spills are emitted in the cold dispatcher below. This is + // also the shape used by the dynasm sibling's pending guard tokens. + sink.block(BlockType::Empty); // A $hot_exit if key_dispatch { - // block $exit (A) — guard/Finish exits br here -> epilogue. // Per resumable label j (opened outermost = the loop header): // block $past_loader_j (B_j) — the fall-through path br's over the // label-j resume loader. // block $loader_j (C_j) — the `br_table` lands here (its end) for // key j+1: the label-j resume loader. // block $dispatch (D) — key 0 br's here: run from the entry. - sink.block(BlockType::Empty); // A $exit for _ in 0..num_labels { sink.block(BlockType::Empty); // B_j (j descending) sink.block(BlockType::Empty); // C_j @@ -2157,14 +2151,6 @@ fn build_function( } } - // Non-key_dispatch loop: the single exit block A (preamble + body share it). - // key_dispatch already opened A/B/C above. A Label-less dispatch trace (CA - // loop, or a bridge chaining nested sub-bridges) also opens A so its - // guard/Finish exits `br` out to the epilogue. - if (has_loop || straightline_dispatch) && !key_dispatch { - sink.block(BlockType::Empty); - } - // Seed with the fail-index base so each guard/finish exit writes // `base + local` into `frame[0]` (every trace passes the next free index // of the global fail-index space, `failguard::fail_descr_base`). The local @@ -2247,18 +2233,18 @@ fn build_function( // a segment that still has `num_labels - labels_passed` labels ahead // sits inside that many (B_j, C_j) pairs, so it br's to depth // `2 * remaining`; the body is unchanged at 1 (every pair closes - // before the loop). `None` for straight-line traces (no block - // emitted). + // before the loop). Straight-line traces use the universal hot exit + // block at depth 0. let block_exit_depth = match (has_loop, in_loop_body) { - // Label-less dispatch trace: one exit block A (depth 0), no `loop`. - (false, _) if straightline_dispatch => Some(0u32), - (false, _) => None, - (true, false) => Some(if key_dispatch { - 2 * (num_labels - labels_passed) as u32 - } else { - 0u32 - }), - (true, true) => Some(1u32), + (false, _) => 0u32, + (true, false) => { + if key_dispatch { + 2 * (num_labels - labels_passed) as u32 + } else { + 0u32 + } + } + (true, true) => 1u32, }; // The guard whose condition the previous op already pushed and tested. // `block_exit_depth` is unchanged across the pair: only a LABEL moves @@ -2277,14 +2263,14 @@ fn build_function( ref_homes, ) { Some(guard) => { - push_cond(&mut sink, constants, value_types, op, kind); - // GUARD_TRUE exits when the condition is false. `i32.eqz` - // rather than the inverted comparison: `!(a < b)` is - // `a >= b` for integers but not for floats, where an - // unordered operand makes both forms false. - if guard.opcode == OpCode::GuardTrue { - sink.i32_eqz(); - } + push_guard_failure_cond( + &mut sink, + constants, + value_types, + op, + kind, + guard.opcode, + ); emit_guard_if_exit( &mut sink, constants, @@ -2419,10 +2405,9 @@ fn build_function( } OpCode::Finish => { - emit_guard_exit(&mut sink, constants, value_types, guard_idx, op); - if let Some(d) = block_exit_depth { - sink.br(d); - } + sink.i32_const(guard_idx as i32); + sink.local_set(bridge_slot_local); + sink.br(block_exit_depth); guard_idx += 1; } @@ -3438,10 +3423,9 @@ fn build_function( } OpCode::GuardFutureCondition | OpCode::GuardAlwaysFails => { // GuardAlwaysFails always exits. - emit_guard_exit(&mut sink, constants, value_types, guard_idx, op); - if let Some(d) = block_exit_depth { - sink.br(d); - } + sink.i32_const(guard_idx as i32); + sink.local_set(bridge_slot_local); + sink.br(block_exit_depth); guard_idx += 1; } @@ -3604,10 +3588,13 @@ fn build_function( & !7; if let (Some(base), Some(inline)) = (residual_type_base, ca.inline) { // rewrite.py's nursery fast path plus assembler.py's inline - // shadow-stack header. The `memory.fill` is deliberate: - // home slots are read as roots before every definition, and - // guard/deopt fail slots may be read by the host. Do not rely - // on nursery reset's pre-existing memset for either class. + // shadow-stack header. The wasm nursery is allocated + // zeroed and its target-specific reset zeroes the complete + // arena before reuse (`Nursery::reset`); the slow born-old + // path also clears its payload. Do not repeat a full-frame + // `memory.fill` on every CALL_ASSEMBLER. The callee prologue + // still explicitly clears its Ref homes, and the fixed + // JitFrame metadata fields are initialized below. sink.i32_const(inline.nursery_free_addr as i32); sink.i32_load(mem32(0)); sink.local_tee(alloc_scratch_local); @@ -3642,14 +3629,8 @@ fn build_function( sink.local_get(alloc_scratch_local); sink.i64_const(inline.jitframe_tid as i64); sink.i64_store(mem64(0)); - // Explicitly initialise the complete JitFrame payload and - // item area, then replicate JitFrame::init + jf_gcmap. - sink.local_get(alloc_scratch_local); - sink.i32_const(GcHeader::SIZE as i32); - sink.i32_add(); - sink.i32_const(0); - sink.i32_const(ca_payload_size as i32); - sink.memory_fill(0); + // Replicate JitFrame::init + jf_gcmap on the already-zero + // payload. Every nonzero metadata field is written below. for offset in [ majit_backend::jitframe::JF_FRAME_INFO_OFS, majit_backend::jitframe::JF_DESCR_OFS, @@ -4780,9 +4761,46 @@ fn build_function( if has_loop { sink.end(); // end loop - sink.end(); // end block - } else if straightline_dispatch { - sink.end(); // end exit block A (Label-less dispatch trace, no `loop`) + } + // A well-formed trace exits through a guard or Finish. Preserve the old + // malformed/natural-fallthrough behavior without letting it enter the cold + // dispatcher with an uninitialized selector. + sink.local_get(0); + sink.return_(); + sink.end(); // end A $hot_exit + + // Structured-Wasm equivalent of the native backends' out-of-line guard + // recovery stubs. Locals retain their fail-site SSA values across the + // branch, so each handler can perform the original positional spills here + // without adding hot-path frame traffic or GC roots. + let cold_exits: Vec<&Op> = ops + .iter() + .filter(|op| op.opcode.is_guard() || op.opcode == OpCode::Finish) + .collect(); + if !cold_exits.is_empty() { + let exit_count = cold_exits.len() as u32; + sink.block(BlockType::Empty); // $cold_done + for _ in cold_exits.iter().rev() { + sink.block(BlockType::Empty); + } + sink.local_get(bridge_slot_local); + if fail_index_base != 0 { + sink.i32_const(fail_index_base as i32); + sink.i32_sub(); + } + sink.br_table((0..exit_count).collect::>(), exit_count); + for (ordinal, op) in cold_exits.into_iter().enumerate() { + sink.end(); + emit_guard_exit( + &mut sink, + constants, + value_types, + fail_index_base + ordinal as u32, + op, + ); + sink.br(exit_count - ordinal as u32 - 1); + } + sink.end(); // end $cold_done } // Epilogue bridge dispatch. Control reaches here only @@ -5182,7 +5200,7 @@ fn emit_guard_true( value_types: &[ValType], guard_idx: u32, op: &Op, - block_exit_depth: Option, + block_exit_depth: u32, ) { emit_resolve(sink, constants, value_types, op.arg(0).to_opref()); sink.i64_eqz(); @@ -5202,7 +5220,7 @@ fn emit_guard_false( value_types: &[ValType], guard_idx: u32, op: &Op, - block_exit_depth: Option, + block_exit_depth: u32, ) { emit_resolve(sink, constants, value_types, op.arg(0).to_opref()); sink.i64_const(0); @@ -5277,36 +5295,29 @@ fn next_op_can_accept_cc<'a>( Some(next_op) } -/// Common guard exit: condition is on stack (i32), emit if + exit. +/// Common guard exit: condition is on stack (i32), emit a tiny hot failure arm. /// /// `block_exit_depth` is the statement-level depth of the enclosing exit /// `block` (preamble = 0, loop body = 1); the `+ 1` accounts for the `if` -/// this opens. `None` for straight-line traces with no exit block. +/// this opens. The actual fail-argument recovery is emitted out of line by +/// `build_function`, matching x86's `write_pending_failure_recoveries`. fn emit_guard_if_exit( sink: &mut InstructionSink<'_>, - constants: &indexmap::IndexMap, + _constants: &indexmap::IndexMap, value_types: &[ValType], guard_idx: u32, - op: &Op, - block_exit_depth: Option, + _op: &Op, + block_exit_depth: u32, ) { sink.if_(BlockType::Empty); - emit_guard_exit(sink, constants, value_types, guard_idx, op); - match block_exit_depth { - // Loop traces: `br` out of this `if` and the enclosing exit `block` - // (the `+ 1` accounts for the `if`) to the function epilogue. - Some(d) => { - sink.br(d + 1); - } - // Straight-line traces have no enclosing block, so fall-through would - // reach the terminal Finish and overwrite frame[0] with its - // fail_index, discarding this guard's exit. Return the frame pointer - // directly (the epilogue's value) to hand control to the metainterp. - None => { - sink.local_get(0); - sink.return_(); - } - } + // The bridge-slot scratch is the first i32 local after the value locals, + // the UintMulHigh scratch locals, and the overflow flag. It is dead until + // the bridge epilogue overwrites it with `local.tee`, so it doubles as the + // cold-exit selector without growing the local or GC-root set. + let exit_selector_local = value_types.len() as u32 + UMULHI_SCRATCH + 2; + sink.i32_const(guard_idx as i32); + sink.local_set(exit_selector_local); + sink.br(block_exit_depth + 1); sink.end(); } @@ -5513,8 +5524,18 @@ fn emit_ovf_binop( } let result_local = 1 + vi; - emit_resolve(sink, constants, value_types, op.arg(0).to_opref()); - emit_resolve(sink, constants, value_types, op.arg(1).to_opref()); + if matches!(binop, BinOp::I64Mul) { + // Keep both factors in the umulhi scratch bank. The hot signed-32 + // overflow check below reuses them without resolving the SSA operands + // again; the slow 64-bit path is free to overwrite the same bank. + emit_resolve(sink, constants, value_types, op.arg(0).to_opref()); + sink.local_tee(num_vars + 1); + emit_resolve(sink, constants, value_types, op.arg(1).to_opref()); + sink.local_tee(num_vars + 2); + } else { + emit_resolve(sink, constants, value_types, op.arg(0).to_opref()); + emit_resolve(sink, constants, value_types, op.arg(1).to_opref()); + } apply_binop(sink, binop); sink.local_set(result_local); @@ -5546,6 +5567,26 @@ fn emit_ovf_binop( sink.local_set(ovf_flag_local); } BinOp::I64Mul => { + // Multiplying two signed-32-bit integers cannot overflow i64: the + // largest magnitude is 2^62. This is the common Python-loop shape + // (e.g. nested_loop's 0..19999 counters), and avoids expanding + // every multiplication into a software 64x64->128 product. The + // exact sign-extension checks preserve the full-width slow path + // for every value outside that proven-safe domain. + sink.local_get(num_vars + 1); + sink.i64_extend32_s(); + sink.local_get(num_vars + 1); + sink.i64_eq(); + sink.local_get(num_vars + 2); + sink.i64_extend32_s(); + sink.local_get(num_vars + 2); + sink.i64_eq(); + sink.i32_and(); + sink.if_(BlockType::Empty); + sink.i64_const(0); + sink.local_set(ovf_flag_local); + sink.else_(); + // Convert the unsigned high word to the signed high word: // smulhi = umulhi - ((a >>s 63) & b) - ((b >>s 63) & a). let high_local = num_vars + 1; @@ -5569,6 +5610,7 @@ fn emit_ovf_binop( sink.i64_ne(); sink.i64_extend_i32_u(); sink.local_set(ovf_flag_local); + sink.end(); } _ => unreachable!("overflow emitter requires add, sub, or mul"), } @@ -5730,6 +5772,49 @@ fn push_cond( } } +/// Push whether a fused GuardTrue/GuardFalse fails. Native backends invert the +/// integer condition code in place (`x86/assembler.py:1778-1784`); spelling the +/// inverse Wasm comparison directly avoids materialising `cmp; i32.eqz` at the +/// hot guard site. Float ordered comparisons deliberately keep `i32.eqz`: +/// their apparent inverse is not equivalent for NaN/unordered operands. +fn push_guard_failure_cond( + sink: &mut InstructionSink<'_>, + constants: &indexmap::IndexMap, + value_types: &[ValType], + op: &Op, + kind: CondKind, + guard_opcode: OpCode, +) { + if guard_opcode == OpCode::GuardFalse { + push_cond(sink, constants, value_types, op, kind); + return; + } + debug_assert_eq!(guard_opcode, OpCode::GuardTrue); + let inverse = match kind { + CondKind::Int(cmp) => Some(CondKind::Int(match cmp { + CmpOp::I64LtS => CmpOp::I64GeS, + CmpOp::I64LeS => CmpOp::I64GtS, + CmpOp::I64Eq => CmpOp::I64Ne, + CmpOp::I64Ne => CmpOp::I64Eq, + CmpOp::I64GtS => CmpOp::I64LeS, + CmpOp::I64GeS => CmpOp::I64LtS, + CmpOp::I64LtU => CmpOp::I64GeU, + CmpOp::I64LeU => CmpOp::I64GtU, + CmpOp::I64GtU => CmpOp::I64LeU, + CmpOp::I64GeU => CmpOp::I64LtU, + })), + CondKind::IsTrue => Some(CondKind::IsZero), + CondKind::IsZero => Some(CondKind::IsTrue), + CondKind::Float(_) => None, + }; + if let Some(inverse) = inverse { + push_cond(sink, constants, value_types, op, inverse); + } else { + push_cond(sink, constants, value_types, op, kind); + sink.i32_eqz(); + } +} + fn emit_cond( sink: &mut InstructionSink<'_>, constants: &indexmap::IndexMap, diff --git a/majit/majit-backend-wasm/tests/codegen_test.rs b/majit/majit-backend-wasm/tests/codegen_test.rs index ad3014f6c55..44e751ece24 100644 --- a/majit/majit-backend-wasm/tests/codegen_test.rs +++ b/majit/majit-backend-wasm/tests/codegen_test.rs @@ -163,6 +163,108 @@ fn raise_catch_clear_root_does_not_cross_the_host_per_exception() { ); } +#[test] +#[ignore = "runtime integration test: needs the release pyre-dynasm, pyre-wasm-runner, and wasm-host module; \ + run via `cargo test -- --ignored` in the check.py job, which builds them"] +fn recursive_call_assembler_does_not_refill_zeroed_nursery_frames() { + let root = workspace_root(); + let dynasm = root.join("target/release/pyre-dynasm"); + let wasm_runner = root.join("target/release/pyre-wasm-runner"); + let host_module = root.join("target/wasm32-unknown-unknown/release/pyre_wasm.wasm-host.wasm"); + let plain_module = root.join("target/wasm32-unknown-unknown/release/pyre_wasm.wasm"); + let wasm_module = if host_module.exists() { + host_module + } else { + plain_module + }; + let script = root.join("pyre/bench/fib_recursive.py"); + + for artifact in [&dynasm, &wasm_runner, &wasm_module] { + assert!( + artifact.exists(), + "runtime recursive-CA regression needs {}; build the requested artifacts first", + artifact.display() + ); + } + let dynasm_run = run_runtime_program(&dynasm, &script, &[]); + assert!(dynasm_run.status.success(), "dynasm recursive fib failed"); + let module = wasm_module.to_str().expect("workspace paths must be UTF-8"); + let wasm_run = run_runtime_program( + &wasm_runner, + &script, + &[ + ("PYRE_WASM_MODULE", module), + ("PYRE_WASM_ENGINE", "wasmtime"), + ("PYRE_WASM_JIT_STATS", "1"), + ("PYRE_WASM_DUMP_ALL_TRACES", "1"), + ("PYRE_WASM_NO_CACHE", "1"), + ], + ); + let stderr = String::from_utf8_lossy(&wasm_run.stderr); + assert!( + wasm_run.status.success(), + "wasm recursive fib failed:\n{stderr}" + ); + assert_eq!( + wasm_run.stdout, dynasm_run.stdout, + "wasm recursive fib output diverged from dynasm:\n{stderr}" + ); + assert_eq!(stat_value(&stderr, "compiles"), 4); + assert_eq!(stat_value(&stderr, "BRIDGE_OK"), 3); + assert!( + !stderr.contains("memory.fill"), + "recursive CA still refills a nursery that is already zeroed:\n{stderr}" + ); +} + +#[test] +#[ignore = "runtime integration test: needs the release pyre-dynasm, pyre-wasm-runner, and wasm-host module; \ + run via `cargo test -- --ignored` in the check.py job, which builds them"] +fn fannkuch_blackhole_helpers_do_not_reflect_through_the_host() { + let root = workspace_root(); + let dynasm = root.join("target/release/pyre-dynasm"); + let wasm_runner = root.join("target/release/pyre-wasm-runner"); + let host_module = root.join("target/wasm32-unknown-unknown/release/pyre_wasm.wasm-host.wasm"); + let plain_module = root.join("target/wasm32-unknown-unknown/release/pyre_wasm.wasm"); + let wasm_module = if host_module.exists() { + host_module + } else { + plain_module + }; + let script = root.join("pyre/bench/fannkuch.py"); + for artifact in [&dynasm, &wasm_runner, &wasm_module] { + assert!( + artifact.exists(), + "runtime fannkuch regression needs {}; build the requested artifacts first", + artifact.display() + ); + } + + let dynasm_run = run_runtime_program(&dynasm, &script, &[]); + assert!(dynasm_run.status.success(), "dynasm fannkuch failed"); + let module = wasm_module.to_str().expect("workspace paths must be UTF-8"); + let wasm_run = run_runtime_program( + &wasm_runner, + &script, + &[ + ("PYRE_WASM_MODULE", module), + ("PYRE_WASM_ENGINE", "wasmtime"), + ("PYRE_WASM_JIT_STATS", "1"), + ], + ); + let stderr = String::from_utf8_lossy(&wasm_run.stderr); + assert!(wasm_run.status.success(), "wasm fannkuch failed:\n{stderr}"); + assert_eq!( + wasm_run.stdout, dynasm_run.stdout, + "wasm fannkuch output diverged from dynasm:\n{stderr}" + ); + assert_eq!(stat_value(&stderr, "compiles"), 28); + assert!( + stat_value(&stderr, "jit_calls") < 100, + "uniform-i64 blackhole helpers still reflected through the host:\n{stderr}" + ); +} + #[test] #[ignore = "runtime integration test: needs the release pyre-dynasm, pyre-wasm-runner, and wasm-host module; \ run via `cargo test -- --ignored` in the check.py job, which builds them"] @@ -461,14 +563,246 @@ fn test_int_sub_ovf_guards_overflow() { #[test] fn test_int_mul_ovf_guards_overflow() { - for (a, b, expected) in [(6, 7, 42), (i64::MIN, 1, i64::MIN), (-9, -7, 63)] { + for (a, b, expected) in [ + (6, 7, 42), + (-9, -7, 63), + (i32::MIN as i64, i32::MIN as i64, 1_i64 << 62), + ( + i32::MAX as i64, + i32::MAX as i64, + (i32::MAX as i64) * (i32::MAX as i64), + ), + (i64::MIN, 1, i64::MIN), + (i32::MAX as i64 + 1, 2, 1_i64 << 32), + ] { assert_eq!(execute_ovf_trace(OpCode::IntMulOvf, a, b), (1, expected)); } - for (a, b) in [(i64::MIN, -1), (i64::MAX, 2), (1_i64 << 62, 3)] { + for (a, b) in [ + (i64::MIN, -1), + (i64::MAX, 2), + (1_i64 << 62, 3), + (i32::MAX as i64 + 1, 1_i64 << 32), + ] { assert_eq!(execute_ovf_trace(OpCode::IntMulOvf, a, b).0, 0); } } +#[test] +fn test_int_mul_ovf_emits_signed32_fast_path_and_full_width_fallback() { + let inputargs = vec![ + InputArg::from_type(Type::Int, 0), + InputArg::from_type(Type::Int, 1), + ]; + let guard = Op::new(OpCode::GuardNoOverflow, &[]); + guard.setfailargs(smallvec![rb(OpRef::input_arg_int(0))]); + let ops = vec![ + make_op( + OpCode::IntMulOvf, + &[OpRef::input_arg_int(0), OpRef::input_arg_int(1)], + OpRef::int_op(2), + ), + guard, + Op::new(OpCode::Finish, &[rb(OpRef::int_op(2))]), + ]; + let (bytes, _) = build_module_default(&inputargs, &ops, &indexmap::IndexMap::new()); + validate_wasm(&bytes); + + let mut extend32_s = 0; + let mut i64_mul = 0; + for payload in wasmparser::Parser::new(0).parse_all(&bytes) { + if let wasmparser::Payload::CodeSectionEntry(body) = payload.unwrap() { + let mut operators = body.get_operators_reader().unwrap(); + while !operators.eof() { + match operators.read().unwrap() { + wasmparser::Operator::I64Extend32S => extend32_s += 1, + wasmparser::Operator::I64Mul => i64_mul += 1, + _ => {} + } + } + } + } + assert_eq!(extend32_s, 2, "both factors need an exact signed-32 check"); + assert!( + i64_mul > 1, + "the software full-width overflow fallback must remain in the module" + ); +} + +/// PyPy's native backends append guard recovery stubs after the hot trace. +/// Keep fail-argument stores out of the successful guard arm in Wasm too: a +/// selector branch should reach one cold `br_table` dispatcher, where the +/// original int/ref/float-bit spills are performed. +#[test] +fn test_guard_recovery_is_out_of_line() { + let inputargs = vec![ + InputArg::from_type(Type::Int, 0), + InputArg::from_type(Type::Ref, 1), + InputArg::from_type(Type::Float, 2), + ]; + let fail_args = smallvec![ + rb(OpRef::input_arg_int(0)), + rb(OpRef::input_arg_ref(1)), + rb(OpRef::input_arg_float(2)), + ]; + let guard = Op::new(OpCode::GuardTrue, &[rb(OpRef::input_arg_int(0))]); + guard.setfailargs(fail_args.clone()); + let finish = Op::new(OpCode::Finish, &fail_args); + finish.setfailargs(fail_args); + let (bytes, guards) = + build_module_default(&inputargs, &[guard, finish], &indexmap::IndexMap::new()); + validate_wasm(&bytes); + assert_eq!(guards.len(), 2); + + let mut control_is_if = Vec::new(); + let mut stores_inside_if = 0; + let mut br_tables = 0; + for payload in wasmparser::Parser::new(0).parse_all(&bytes) { + if let wasmparser::Payload::CodeSectionEntry(body) = payload.unwrap() { + let mut operators = body.get_operators_reader().unwrap(); + while !operators.eof() { + match operators.read().unwrap() { + wasmparser::Operator::If { .. } => control_is_if.push(true), + wasmparser::Operator::Block { .. } | wasmparser::Operator::Loop { .. } => { + control_is_if.push(false); + } + wasmparser::Operator::End => { + control_is_if.pop(); + } + wasmparser::Operator::I64Store { .. } + if control_is_if.iter().any(|inside_if| *inside_if) => + { + stores_inside_if += 1; + } + wasmparser::Operator::BrTable { .. } => br_tables += 1, + _ => {} + } + } + } + } + assert_eq!(stores_inside_if, 0, "guard arms must not spill fail args"); + assert_eq!(br_tables, 1, "all exits must share one cold dispatcher"); +} + +#[test] +fn test_fused_integer_guard_true_uses_inverse_comparison_directly() { + let inputargs = vec![ + InputArg::from_type(Type::Int, 0), + InputArg::from_type(Type::Int, 1), + ]; + let compare = make_op( + OpCode::IntLt, + &[OpRef::input_arg_int(0), OpRef::input_arg_int(1)], + OpRef::int_op(2), + ); + let guard = make_guard( + OpCode::GuardTrue, + &[OpRef::int_op(2)], + &[OpRef::input_arg_int(0)], + ); + let finish = Op::new(OpCode::Finish, &[rb(OpRef::input_arg_int(1))]); + let (bytes, _) = build_module_default( + &inputargs, + &[compare, guard, finish], + &indexmap::IndexMap::new(), + ); + validate_wasm(&bytes); + + let mut ge_s = 0; + let mut i32_eqz = 0; + for payload in wasmparser::Parser::new(0).parse_all(&bytes) { + if let wasmparser::Payload::CodeSectionEntry(body) = payload.unwrap() { + let mut operators = body.get_operators_reader().unwrap(); + while !operators.eof() { + match operators.read().unwrap() { + wasmparser::Operator::I64GeS => ge_s += 1, + wasmparser::Operator::I32Eqz => i32_eqz += 1, + _ => {} + } + } + } + } + assert_eq!(ge_s, 1, "IntLt guard failure should be emitted as IntGe"); + assert_eq!(i32_eqz, 0, "integer inverse must not materialize i32.eqz"); +} + +#[test] +fn test_cold_guard_recovery_preserves_nonzero_base_and_typed_bits() { + const FAIL_INDEX_BASE: u32 = 37; + let inputargs = vec![ + InputArg::from_type(Type::Int, 0), + InputArg::from_type(Type::Ref, 1), + InputArg::from_type(Type::Float, 2), + ]; + let fail_args = smallvec![ + rb(OpRef::input_arg_ref(1)), + rb(OpRef::input_arg_float(2)), + rb(OpRef::input_arg_int(0)), + ]; + let guard = Op::new(OpCode::GuardTrue, &[rb(OpRef::input_arg_int(0))]); + guard.setfailargs(fail_args.clone()); + let finish = Op::new(OpCode::Finish, &fail_args); + finish.setfailargs(fail_args); + let ops = [guard, finish]; + let (bytes, guards, _, _, _) = codegen::build_wasm_module( + &inputargs, + &ops, + &indexmap::IndexMap::new(), + Some(0), + &HashMap::new(), + &codegen::GuardGcTypeInfo::default(), + codegen::AllocHelpers::default(), + 0, + None, + 0, + 0, + FAIL_INDEX_BASE, + 0, + 0, + codegen::FrameGeometry::fixed(), + codegen::CaParams::default(), + ) + .expect("wasm codegen should succeed"); + assert_eq!(guards[0].fail_index, FAIL_INDEX_BASE); + + let ref_bits = 0x1234_5678_i64; + let float_bits = (-13.25_f64).to_bits() as i64; + let mut store = Store::new(&Engine::default(), ()); + let engine = store.engine().clone(); + let module = Module::new(&engine, &bytes).expect("generated trace should compile"); + let memory = Memory::new(&mut store, MemoryType::new(1, None)).unwrap(); + for (slot, bits) in [0_i64, ref_bits, float_bits].into_iter().enumerate() { + memory + .write( + &mut store, + codegen::FRAME_SLOT_BASE as usize + slot * 8, + &bits.to_le_bytes(), + ) + .unwrap(); + } + let mut linker = Linker::new(&engine); + linker.define("env", "memory", memory).unwrap(); + let instance = linker.instantiate_and_start(&mut store, &module).unwrap(); + instance + .get_typed_func::(&store, "trace") + .unwrap() + .call(&mut store, 0) + .unwrap(); + + let mut word = [0; 8]; + memory.read(&store, 0, &mut word).unwrap(); + assert_eq!(i64::from_le_bytes(word), FAIL_INDEX_BASE as i64); + for (slot, expected) in [ref_bits, float_bits, 0].into_iter().enumerate() { + memory + .read( + &store, + codegen::FRAME_SLOT_BASE as usize + slot * 8, + &mut word, + ) + .unwrap(); + assert_eq!(i64::from_le_bytes(word), expected); + } +} + #[test] fn test_guard_overflow_uses_pending_flag() { assert_eq!( diff --git a/pyre/pyre-wasm/src/lib.rs b/pyre/pyre-wasm/src/lib.rs index 5262066dd6d..9af1426202f 100644 --- a/pyre/pyre-wasm/src/lib.rs +++ b/pyre/pyre-wasm/src/lib.rs @@ -162,12 +162,56 @@ mod residual_host { unsafe impl Sync for Scratch {} static SCRATCH: Scratch = Scratch(UnsafeCell::new([0u8; SCRATCH_LEN])); + /// Direct-call the small family of blackhole helpers whose real wasm ABI + /// is exactly the uniform `i64` signature carried by the residual call. + /// + /// The generic path below must reflect the callee's wasm type in the host: + /// an `r` argument may be a real `i32` pointer, a void descriptor may name + /// a word-returning target, and guessing either signature traps at a wasm + /// `call_indirect`. These five targets are different: their declarations + /// are explicit `extern "C" fn(i64, ...) -> i64` wrappers, and the CPU + /// function table stores those exact function addresses. Comparing the + /// table index (`fn as usize` on wasm32) therefore proves both the callee + /// identity and its ABI. Calling the named wrapper directly matches the + /// native blackhole dispatch while avoiding a guest -> host -> guest + /// reflection round-trip. + /// + /// Keep this as an exact-function allow-list, not a signature inference. + /// A mismatched arity deliberately falls through to the reflective path. + fn direct_uniform_i64_call(func_ptr: usize, args: &[i64]) -> Option { + match args { + [value] if func_ptr == pyre_jit::call_jit::bh_box_int_fn as usize => { + Some(pyre_jit::call_jit::bh_box_int_fn(*value)) + } + [value] if func_ptr == pyre_jit::call_jit::bh_truth_fn as usize => { + Some(pyre_jit::call_jit::bh_truth_fn(*value)) + } + [lhs, rhs, op_code] if func_ptr == pyre_jit::call_jit::bh_binary_op_fn as usize => { + Some(pyre_jit::call_jit::bh_binary_op_fn(*lhs, *rhs, *op_code)) + } + [lhs, rhs, op_code] if func_ptr == pyre_jit::call_jit::bh_compare_fn as usize => { + Some(pyre_jit::call_jit::bh_compare_fn(*lhs, *rhs, *op_code)) + } + [obj, key, value] + if func_ptr == pyre_interpreter::opcode_ops::bh_store_subscr_fn as usize => + { + Some(pyre_interpreter::opcode_ops::bh_store_subscr_fn( + *obj, *key, *value, + )) + } + _ => None, + } + } + fn residual_host_call(func_ptr: usize, args: &[i64]) -> i64 { assert!( args.len() <= MAX_ARGS, "residual_host_call: arity {} exceeds {MAX_ARGS}", args.len() ); + if let Some(result) = direct_uniform_i64_call(func_ptr, args) { + return result; + } let base = SCRATCH.0.get() as *mut u8; unsafe { (base.add(CALL_FUNC_OFS) as *mut i64).write_unaligned(func_ptr as i64);