diff --git a/crates/metaltile-codegen/src/cuda/mod.rs b/crates/metaltile-codegen/src/cuda/mod.rs index 83edea65..dccf899c 100644 --- a/crates/metaltile-codegen/src/cuda/mod.rs +++ b/crates/metaltile-codegen/src/cuda/mod.rs @@ -62,7 +62,7 @@ use crate::{ /// Shared preamble for every emitted CUDA kernel: headers, the `MT_INF` /// constant (NVRTC has no `INFINITY` macro), and the block-scaled-quant /// decode helpers ported verbatim from `msl/preamble.rs` (pure arithmetic -/// → identical to `quant::codec`). Unused helpers are pruned by the +/// → identical to `kernels::quant::codec`). Unused helpers are pruned by the /// compiler, so we always emit them. const CUDA_PREAMBLE: &str = r#"// Generated by MetalTile (CUDA backend). DO NOT EDIT. #include diff --git a/crates/metaltile-std/src/ffai/mod.rs b/crates/metaltile-std/src/ffai/mod.rs deleted file mode 100644 index ecb1ca01..00000000 --- a/crates/metaltile-std/src/ffai/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric -//! SPDX-License-Identifier: Apache-2.0 -//! FFAI / model-specific kernels. -//! -//! Kernels here are ports from FFAI / mlx-swift-lm / ekryski's `mlx` fork -//! that don't have a matching template in mainline MLX at the pinned -//! commit (see `metaltile-std/build.rs` — `MLX_COMMIT`). They register -//! a `BenchSpec` so `tile build` / `tile inspect` can find them, but -//! the spec uses `shapes: &[]` and `dispatch: BenchDispatch::Generic`, -//! so `tile bench` skips them (no MLX side-by-side, no GPU shapes). -//! -//! Correctness for these kernels is validated end-to-end in FFAI's -//! integration tests against real models. Once a kernel has been -//! verified there, the shape spec / bench dispatch can be added back -//! here so `tile bench` can track it for regressions — and if its MLX -//! counterpart lands in mainline at a future pin, the file moves to -//! `mlx/`. - -pub mod attn_head_gate; -pub mod aura_dequant_rotated; -pub mod aura_encode; -pub mod aura_flash_p1; -pub mod aura_flash_pass2; -pub mod aura_flash_sdpa; -pub mod aura_score; -pub mod aura_value; -// batched_* projection GEMV/GEMM + dequant_gemv migrated to kernels/gemm/. -pub mod dequant_gather; -pub mod dequant_gather_block_scaled; -pub mod dsv4_compressor_pool; -pub mod dsv4_csa_sdpa_decode; -pub mod dsv4_fp8_block_dequant; -pub mod dsv4_indexer_score; -pub mod dsv4_indexer_topk; -pub mod dsv4_mhc; -pub mod dsv4_mhc_sinkhorn_split; -pub mod dsv4_mxfp4_dequant; -pub mod dsv4_swiglu_limit; -pub mod ffai_dequant_q4; -pub mod flash_block_scaled_sdpa; -pub mod flash_quantized_sdpa; -pub mod gate_up_swiglu_fused; -pub mod gelu_erf; -// gemm_q4_mpp / gemm_q8 / gemm_q8_mpp + gemv_q8 (split by family) → kernels/. -pub mod gguf_dequant_iq2_xxs; -pub mod gguf_dequant_iq2_xxs_raw; -pub mod gguf_dequant_q2_k; -pub mod gguf_dequant_q8_0; -pub mod gguf_iq2_xxs_extract_qs; -pub mod leaky_relu; -// moe family (moe* · dsv4_router_topk · dequant_gemv_expert_indexed*) → -// kernels/moe/. patch_embed_block_scaled / patch_embed_mma_block_scaled → -// kernels/gemm/. -pub mod sdpa_bidirectional; -pub mod sdpa_bidirectional_d128_relpos; -pub mod sdpa_bidirectional_windowed; -pub mod sdpa_decode; // unified: d64, d96, d128, d256, d512 -pub mod sdpa_decode_2pass; -pub mod sdpa_decode_batched; -pub mod sdpa_decode_d512_sink; -pub mod sdpa_decode_sink_buf; -pub mod sdpa_multi; -pub mod sdpa_multi_d256; -pub mod sdpa_prefill_d512_sink; -pub mod sdpa_rel_pos_conformer; diff --git a/crates/metaltile-std/src/kernels/convolution/consolidated/conv1d.rs b/crates/metaltile-std/src/kernels/convolution/consolidated/conv1d.rs index c6972df2..2917959e 100644 --- a/crates/metaltile-std/src/kernels/convolution/consolidated/conv1d.rs +++ b/crates/metaltile-std/src/kernels/convolution/consolidated/conv1d.rs @@ -383,7 +383,7 @@ pub fn mt_conv1d_quant( #[allow(clippy::too_many_arguments)] fn blockscaled_setup( kernel: metaltile::core::ir::Kernel, - fmt: crate::quant::format::QFormat, + fmt: crate::kernels::quant::format::QFormat, batch: usize, in_ch: usize, in_len: usize, @@ -406,8 +406,8 @@ fn blockscaled_setup( let bias_f: Vec = (0..out_ch).map(|i| ((i % 5) as f32 / 5.0 - 0.5) * 2.0).collect(); let weight_f: Vec = (0..(out_ch * c_dim)).map(|i| ((i % 11) as f32 / 11.0 - 0.5) * 4.0).collect(); - let p = crate::quant::format::pack(fmt, &weight_f, out_ch, c_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_ch, c_dim); + let p = crate::kernels::quant::format::pack(fmt, &weight_f, out_ch, c_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_ch, c_dim); let input = crate::utils::unpack_f32(&crate::utils::pack_f32(&input_f, dt), dt); let bias = crate::utils::unpack_f32(&crate::utils::pack_f32(&bias_f, dt), dt); let expected = { @@ -438,8 +438,8 @@ fn blockscaled_setup( }; let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -467,7 +467,7 @@ fn blockscaled_setup( if dilation > 1 { s = s.constexpr("dilation", dilation as u32); } - if matches!(fmt, crate::quant::format::QFormat::Nvfp4) { + if matches!(fmt, crate::kernels::quant::format::QFormat::Nvfp4) { s = s.constexpr("global", p.global); } s.expect(TestBuffer::from_vec("out", crate::utils::pack_f32(&expected, dt), dt)) @@ -911,13 +911,13 @@ pub mod kernel_tests { // Named FMT values in this mini-list: mxfp4=0, nvfp4=1, fp8_e5m2=2, int8=3. // Named DIL values: audio=0, fishspeech=1. let fmt = if FMT == 0u32 { - crate::quant::format::QFormat::Mxfp4 + crate::kernels::quant::format::QFormat::Mxfp4 } else if FMT == 1u32 { - crate::quant::format::QFormat::Nvfp4 + crate::kernels::quant::format::QFormat::Nvfp4 } else if FMT == 2u32 { - crate::quant::format::QFormat::Fp8E5m2 + crate::kernels::quant::format::QFormat::Fp8E5m2 } else { - crate::quant::format::QFormat::Int8 + crate::kernels::quant::format::QFormat::Int8 }; let kernel = if DIL == 0u32 { if FMT == 0u32 { @@ -1021,22 +1021,25 @@ pub mod kernel_benches { // Named FMT values in this mini-list: mxfp4=0, nvfp4=1, fp8_e5m2=2, int8=3. // Named DIL values: audio=0, fishspeech=1. let fmt = if FMT == 0u32 { - crate::quant::format::QFormat::Mxfp4 + crate::kernels::quant::format::QFormat::Mxfp4 } else if FMT == 1u32 { - crate::quant::format::QFormat::Nvfp4 + crate::kernels::quant::format::QFormat::Nvfp4 } else if FMT == 2u32 { - crate::quant::format::QFormat::Fp8E5m2 + crate::kernels::quant::format::QFormat::Fp8E5m2 } else { - crate::quant::format::QFormat::Int8 + crate::kernels::quant::format::QFormat::Int8 }; let (codes_dt, codes_len) = if fmt.element_bits() == 8 { (DType::U8, out_ch * c_dim) } else { - (DType::U32, crate::quant::format::bitstream_words(out_ch * c_dim, fmt.element_bits())) + ( + DType::U32, + crate::kernels::quant::format::bitstream_words(out_ch * c_dim, fmt.element_bits()), + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = out_ch * (c_dim / fmt.block_size()); @@ -1091,7 +1094,7 @@ pub mod kernel_benches { if DIL == 1u32 { s = s.constexpr("dilation", 1u32); } - if matches!(fmt, crate::quant::format::QFormat::Nvfp4) { + if matches!(fmt, crate::kernels::quant::format::QFormat::Nvfp4) { s = s.constexpr("global", 1.0f32); } s.grid_1d(n_out, 256) diff --git a/crates/metaltile-std/src/kernels/convolution/conv2d_block_scaled.rs b/crates/metaltile-std/src/kernels/convolution/conv2d_block_scaled.rs index 3c64f841..f82d1365 100644 --- a/crates/metaltile-std/src/kernels/convolution/conv2d_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/convolution/conv2d_block_scaled.rs @@ -29,7 +29,7 @@ //! taps clamped to contribute zero. `C` is a multiple of `block_size` (4-bit //! `block_size` a multiple of 8). fp8_e4m3 reuses the nvfp8 kernel. Codegen-only; //! correctness pinned by the in-source `#[test_kernel]`s vs a -//! `quant::format::dequant` oracle running the dense conv2d math. +//! `kernels::quant::format::dequant` oracle running the dense conv2d math. use metaltile::kernel; /// Quantized-weight conv2d, folded over the 28-format axis (§7). Geometry is the @@ -176,7 +176,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -277,8 +277,8 @@ pub mod kernel_tests { let bias_f = ramp(out_ch, 5, 2.0); // Quantize the [out_ch, C] filter via the shared codec. let w_f = ramp(out_ch * contraction, 11, 4.0); - let p = crate::quant::format::pack(fmt, &w_f, out_ch, contraction); - let wdq = crate::quant::format::dequant(fmt, &p, out_ch, contraction); + let p = crate::kernels::quant::format::pack(fmt, &w_f, out_ch, contraction); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_ch, contraction); let input = unpack_f32(&pack_f32(&input_f, dt), dt); let bias = unpack_f32(&pack_f32(&bias_f, dt), dt); // Oracle: dense conv2d over the dequantized filter row [out_ch, C]. @@ -292,8 +292,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -382,7 +382,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn conv2d_bench( @@ -409,11 +409,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n_codes, DType::U8) } else { - (crate::quant::format::bitstream_words(n_codes, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(n_codes, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = out_ch * (contraction / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/convolution/conv2d_mma_block_scaled.rs b/crates/metaltile-std/src/kernels/convolution/conv2d_mma_block_scaled.rs index 6ad6b3f9..bf050af4 100644 --- a/crates/metaltile-std/src/kernels/convolution/conv2d_mma_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/convolution/conv2d_mma_block_scaled.rs @@ -350,7 +350,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::{QFormat, ScaleKind}, + kernels::quant::format::{QFormat, ScaleKind}, utils::{pack_f32, unpack_f32}, }; @@ -440,8 +440,8 @@ pub mod kernel_tests { let input_f = ramp(batch * in_ch * in_h * in_w, 13, 2.0); // Quantize the [out_ch, BK] filter via the shared codec. let filter_f = ramp(out_ch * bk, 11, 2.0); - let p = crate::quant::format::pack(fmt, &filter_f, out_ch, bk); - let wdq = crate::quant::format::dequant(fmt, &p, out_ch, bk); + let p = crate::kernels::quant::format::pack(fmt, &filter_f, out_ch, bk); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_ch, bk); let input = unpack_f32(&pack_f32(&input_f, dt), dt); let expected = naive_conv2d_mma(&input, &wdq, batch, in_ch, in_h, in_w, out_ch, kh, kw); // Axis-driven binding (robust across all element widths/scale kinds): @@ -872,7 +872,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::{QFormat, ScaleKind}; + use crate::kernels::quant::format::{QFormat, ScaleKind}; #[allow(clippy::too_many_arguments)] fn mma_bench( @@ -900,7 +900,10 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (out_ch * bk, DType::U8) } else { - (crate::quant::format::bitstream_words(out_ch * bk, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(out_ch * bk, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { ScaleKind::F32 => DType::F32, diff --git a/crates/metaltile-std/src/kernels/convolution/conv3d_block_scaled.rs b/crates/metaltile-std/src/kernels/convolution/conv3d_block_scaled.rs index 70b94832..0c6222fa 100644 --- a/crates/metaltile-std/src/kernels/convolution/conv3d_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/convolution/conv3d_block_scaled.rs @@ -28,7 +28,7 @@ //! taps clamped to contribute zero. `C` is a multiple of `block_size` (4-bit //! `block_size` a multiple of 8). fp8_e4m3 reuses the nvfp8 kernel. //! Codegen-only; correctness pinned by the in-source `#[test_kernel]`s vs a -//! `quant::format::dequant` oracle running the dense conv3d math. +//! `kernels::quant::format::dequant` oracle running the dense conv3d math. use metaltile::kernel; @@ -193,7 +193,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -315,8 +315,8 @@ pub mod kernel_tests { let bias_f = ramp(out_ch, 5, 2.0); // Quantize the [out_ch, C] filter via the shared codec. let w_f = ramp(out_ch * contraction, 11, 4.0); - let p = crate::quant::format::pack(fmt, &w_f, out_ch, contraction); - let wdq = crate::quant::format::dequant(fmt, &p, out_ch, contraction); + let p = crate::kernels::quant::format::pack(fmt, &w_f, out_ch, contraction); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_ch, contraction); let input = unpack_f32(&pack_f32(&input_f, dt), dt); let bias = unpack_f32(&pack_f32(&bias_f, dt), dt); // Oracle: dense conv3d over the dequantized filter row [out_ch, C]. @@ -330,8 +330,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -424,7 +424,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn conv3d_bench( @@ -455,11 +455,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (total, DType::U8) } else { - (crate::quant::format::bitstream_words(total, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(total, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = out_ch * (contraction / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/convolution/conv3d_mma_block_scaled.rs b/crates/metaltile-std/src/kernels/convolution/conv3d_mma_block_scaled.rs index d1459bb0..7a6bad98 100644 --- a/crates/metaltile-std/src/kernels/convolution/conv3d_mma_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/convolution/conv3d_mma_block_scaled.rs @@ -42,7 +42,7 @@ //! //! fp8_e4m3 reuses the nvfp8 kernel (same 8-bit-E4M3 + f32-scale shape). //! Codegen-only; correctness pinned by the in-source `#[test_kernel]`s vs a -//! `quant::format::dequant` oracle running the dense conv3d_mma math. +//! `kernels::quant::format::dequant` oracle running the dense conv3d_mma math. use metaltile::kernel; @@ -356,7 +356,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -461,8 +461,8 @@ pub mod kernel_tests { let input_f = ramp(batch * in_ch * in_d * in_h * in_w, 13, 2.0); // Quantize the [out_ch, C] filter via the shared codec. let w_f = ramp(out_ch * contraction, 11, 2.0); - let p = crate::quant::format::pack(fmt, &w_f, out_ch, contraction); - let wdq = crate::quant::format::dequant(fmt, &p, out_ch, contraction); + let p = crate::kernels::quant::format::pack(fmt, &w_f, out_ch, contraction); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_ch, contraction); let input = unpack_f32(&pack_f32(&input_f, dt), dt); // Oracle: dense conv3d_mma over the dequantized filter row [out_ch, C]. let expected = @@ -473,8 +473,8 @@ pub mod kernel_tests { // int/mxint width routes correctly. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -636,7 +636,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn mma_bench( @@ -666,11 +666,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (total_elems, DType::U8) } else { - (crate::quant::format::bitstream_words(total_elems, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(total_elems, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = out_ch * (contraction / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/convolution/depthwise_conv2d_block_scaled.rs b/crates/metaltile-std/src/kernels/convolution/depthwise_conv2d_block_scaled.rs index 611a84d7..b612520e 100644 --- a/crates/metaltile-std/src/kernels/convolution/depthwise_conv2d_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/convolution/depthwise_conv2d_block_scaled.rs @@ -28,7 +28,7 @@ //! this matrix-coverage kernel we use **`k = 8 → C = 64`** in the tests so all //! nine formats fit the generic codec; the kernel body itself is general over //! `k`. Codegen-only; correctness pinned by the in-source `#[test_kernel]`s vs a -//! `quant::format::dequant` oracle. +//! `kernels::quant::format::dequant` oracle. //! //! ## DISPATCH INVARIANTS //! @@ -174,7 +174,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -208,8 +208,8 @@ pub mod kernel_tests { let input_f = ramp(batch * ch * in_h * in_w, 13, 6.0); let bias_f = ramp(ch, 5, 2.0); let w_f = ramp(ch * cols, 11, 4.0); - let p = crate::quant::format::pack(fmt, &w_f, ch, cols); - let wdq = crate::quant::format::dequant(fmt, &p, ch, cols); + let p = crate::kernels::quant::format::pack(fmt, &w_f, ch, cols); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, ch, cols); let input = unpack_f32(&pack_f32(&input_f, dt), dt); let bias = unpack_f32(&pack_f32(&bias_f, dt), dt); // Oracle: dense depthwise math over the dequantized [ch, C] filter, @@ -248,8 +248,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -442,7 +442,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn dw_bench( @@ -469,11 +469,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (ch * cols, DType::U8) } else { - (crate::quant::format::bitstream_words(ch * cols, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(ch * cols, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = ch * (cols / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qgemv.rs b/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qgemv.rs index 8baef5ad..34f30562 100644 --- a/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qgemv.rs +++ b/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qgemv.rs @@ -390,7 +390,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -424,8 +424,8 @@ pub mod kernel_tests { ) -> TestSetup { let pack_w = |out_dim: usize, seed: usize| { let w = weights(out_dim, in_dim, seed); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); (p, wdq) }; let (pa, wdq_a) = pack_w(out_a, 0); @@ -446,8 +446,8 @@ pub mod kernel_tests { // buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_a.max(out_b).max(out_c).max(out_d); @@ -904,7 +904,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn batched_4_bench( @@ -922,8 +922,8 @@ pub mod kernel_benches { // packs + int2/3/5/6 tight bit-streams) tight-bit-packs into u32 words. let codes_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_a.max(out_b).max(out_c).max(out_d); @@ -935,7 +935,7 @@ pub mod kernel_benches { if elem_bits == 8 { o * in_dim } else { - crate::quant::format::bitstream_words(o * in_dim, elem_bits) + crate::kernels::quant::format::bitstream_words(o * in_dim, elem_bits) } }; let scl = |o: usize| o * (in_dim / bs); diff --git a/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qmm.rs b/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qmm.rs index 133f2871..1dbd933c 100644 --- a/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qmm.rs +++ b/crates/metaltile-std/src/kernels/gemm/batched_4_block_scaled_qmm.rs @@ -404,7 +404,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -460,8 +460,8 @@ pub mod kernel_tests { // Pack + dequant each of the four weight matrices (distinct seeds). let pack_w = |out_dim: usize, seed: usize| { let w = weights(out_dim, in_dim, seed); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); (p, wdq) }; let (pa, wdq_a) = pack_w(out_a, 0); @@ -483,8 +483,8 @@ pub mod kernel_tests { // buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_a.max(out_b).max(out_c).max(out_d); @@ -971,7 +971,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn batched_4_qmm_bench( @@ -992,8 +992,8 @@ pub mod kernel_benches { // right buffer types. let codes_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_a.max(out_b).max(out_c).max(out_d); @@ -1004,7 +1004,7 @@ pub mod kernel_benches { if fmt.element_bits() == 8 { o * in_dim } else { - crate::quant::format::bitstream_words(o * in_dim, fmt.element_bits()) + crate::kernels::quant::format::bitstream_words(o * in_dim, fmt.element_bits()) } }; let scl = |o: usize| o * (in_dim / bs); diff --git a/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qgemv.rs b/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qgemv.rs index 12904a27..6f5f4b2e 100644 --- a/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qgemv.rs +++ b/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qgemv.rs @@ -308,7 +308,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -340,8 +340,8 @@ pub mod kernel_tests { ) -> TestSetup { let pack_w = |out_dim: usize, seed: usize| { let w = weights(out_dim, in_dim, seed); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); (p, wdq) }; let (pq, wdq_q) = pack_w(out_q, 0); @@ -359,8 +359,8 @@ pub mod kernel_tests { // the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_q.max(out_k).max(out_v); @@ -701,7 +701,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn qkv_bench( kernel: Kernel, @@ -717,8 +717,8 @@ pub mod kernel_benches { // packs + int2/3/5/6 tight bit-streams) tight-bit-packs into u32 words. let codes_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_q.max(out_k).max(out_v); @@ -730,7 +730,7 @@ pub mod kernel_benches { if fmt.element_bits() == 8 { o * in_dim } else { - crate::quant::format::bitstream_words(o * in_dim, fmt.element_bits()) + crate::kernels::quant::format::bitstream_words(o * in_dim, fmt.element_bits()) } }; let scl = |o: usize| o * (in_dim / bs); diff --git a/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qmm.rs b/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qmm.rs index 45404225..4a02ce45 100644 --- a/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qmm.rs +++ b/crates/metaltile-std/src/kernels/gemm/batched_qkv_block_scaled_qmm.rs @@ -324,7 +324,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -379,8 +379,8 @@ pub mod kernel_tests { // Pack + dequant each of the three weight matrices (distinct seeds). let pack_w = |out_dim: usize, seed: usize| { let w = weights(out_dim, in_dim, seed); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); (p, wdq) }; let (pq, wdq_q) = pack_w(out_q, 0); @@ -399,8 +399,8 @@ pub mod kernel_tests { // buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_q.max(out_k).max(out_v); @@ -760,7 +760,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn qkv_bench( @@ -778,8 +778,8 @@ pub mod kernel_benches { // packs + int2/3/5/6 tight bit-streams) tight-bit-packs into u32 words. let codes_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let max_rows = out_q.max(out_k).max(out_v); @@ -790,7 +790,7 @@ pub mod kernel_benches { if fmt.element_bits() == 8 { o * in_dim } else { - crate::quant::format::bitstream_words(o * in_dim, fmt.element_bits()) + crate::kernels::quant::format::bitstream_words(o * in_dim, fmt.element_bits()) } }; let scl = |o: usize| o * (in_dim / bs); diff --git a/crates/metaltile-std/src/kernels/gemm/block_scaled_matmul.rs b/crates/metaltile-std/src/kernels/gemm/block_scaled_matmul.rs index 3f0b1381..a2aeb272 100644 --- a/crates/metaltile-std/src/kernels/gemm/block_scaled_matmul.rs +++ b/crates/metaltile-std/src/kernels/gemm/block_scaled_matmul.rs @@ -34,7 +34,7 @@ use metaltile::kernel; /// 2 = E4M3 byte, 3 = E5M2 byte, 4 = int8 byte. /// SKIND 0 = E8M0 pow-2 (u8), 1 = E4M3 micro × global (u8, nvfp4), /// 2 = direct per-block scale (f32 / f16). -/// Decodes through the shared `kernels/primitives.rs` (mirroring `quant::codec`) +/// Decodes through the shared `kernels/primitives.rs` (mirroring `kernels::quant::codec`) /// so the kernel and the oracle cannot drift. Produces `mt__qgemv`. #[kernel(variants( (FMT, BITS, WT, ST, WDEC, SKIND) = [ @@ -170,7 +170,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{block_scaled_qgemv_oracle, block_scaled_weights, pack_f32, unpack_f32}, }; @@ -185,8 +185,8 @@ pub mod kernel_tests { dt: DType, ) -> TestSetup { let w = block_scaled_weights(out_dim, in_dim); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); let input_f: Vec = (0..in_dim).map(|i| ((i % 11) as f32 - 5.0) * 0.01).collect(); // Round-trip the input through `dt` so the oracle sees what the GPU sees. let x = unpack_f32(&pack_f32(&input_f, dt), dt); @@ -198,8 +198,8 @@ pub mod kernel_tests { // buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -369,7 +369,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn qgemv_bench( kernel: Kernel, @@ -385,11 +385,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n, DType::U8) } else { - (crate::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/kernels/gemm/block_scaled_mma.rs b/crates/metaltile-std/src/kernels/gemm/block_scaled_mma.rs index 9bf01219..ffa53d4e 100644 --- a/crates/metaltile-std/src/kernels/gemm/block_scaled_mma.rs +++ b/crates/metaltile-std/src/kernels/gemm/block_scaled_mma.rs @@ -295,7 +295,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -336,8 +336,8 @@ pub mod kernel_tests { dt: DType, ) -> TestSetup { let w = weights(n, k); - let p = crate::quant::format::pack(fmt, &w, n, k); - let wdq = crate::quant::format::dequant(fmt, &p, n, k); + let p = crate::kernels::quant::format::pack(fmt, &w, n, k); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, n, k); let x_f: Vec = (0..m * k).map(|i| ((i % 11) as f32 - 5.0) * 0.01).collect(); let x = unpack_f32(&pack_f32(&x_f, dt), dt); let expected = qmm_oracle(&wdq, &x, m, k, n); @@ -348,8 +348,8 @@ pub mod kernel_tests { // buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -527,7 +527,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn mma_bench( kernel: Kernel, @@ -543,11 +543,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n * k, DType::U8) } else { - (crate::quant::format::bitstream_words(n * k, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n * k, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm.rs b/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm.rs index 9a510fcc..54ce5b58 100644 --- a/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm.rs +++ b/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm.rs @@ -168,7 +168,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{block_scaled_qmm_oracle, block_scaled_weights, pack_f32, unpack_f32}, }; @@ -184,8 +184,8 @@ pub mod kernel_tests { dt: DType, ) -> TestSetup { let w = block_scaled_weights(out_dim, in_dim); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); let x_f: Vec = (0..m_rows * in_dim).map(|i| ((i % 11) as f32 - 5.0) * 0.01).collect(); let x = unpack_f32(&pack_f32(&x_f, dt), dt); let expected = block_scaled_qmm_oracle(&wdq, &x, m_rows, in_dim, out_dim); @@ -194,8 +194,8 @@ pub mod kernel_tests { // FP16 scales as f16; E8M0/E4M3 scales as one byte. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -365,7 +365,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn qmm_bench( kernel: Kernel, @@ -383,11 +383,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n_elems, DType::U8) } else { - (crate::quant::format::bitstream_words(n_elems, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(n_elems, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_mpp.rs b/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_mpp.rs index 985a8f27..3d1dd769 100644 --- a/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_mpp.rs +++ b/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_mpp.rs @@ -191,7 +191,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -212,8 +212,8 @@ pub mod kernel_tests { if i % 3 == 0 { -mag } else { mag } }) .collect(); - let p = crate::quant::format::pack(fmt, &w, n, k); - let wdq = crate::quant::format::dequant(fmt, &p, n, k); + let p = crate::kernels::quant::format::pack(fmt, &w, n, k); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, n, k); let x_f: Vec = (0..m * k).map(|i| ((i % 11) as f32 - 5.0) * 0.02).collect(); let x = unpack_f32(&pack_f32(&x_f, dt), dt); let mut expected = vec![0.0f32; m * n]; @@ -233,8 +233,8 @@ pub mod kernel_tests { // the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -401,7 +401,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn mpp_bench( kernel: Kernel, @@ -417,11 +417,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n * k, DType::U8) } else { - (crate::quant::format::bitstream_words(n * k, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n * k, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = n * (k / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_nax.rs b/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_nax.rs index 815e0191..6639f673 100644 --- a/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_nax.rs +++ b/crates/metaltile-std/src/kernels/gemm/block_scaled_qmm_nax.rs @@ -198,7 +198,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -219,8 +219,8 @@ pub mod kernel_tests { if i % 3 == 0 { -mag } else { mag } }) .collect(); - let p = crate::quant::format::pack(fmt, &w, n, k); - let wdq = crate::quant::format::dequant(fmt, &p, n, k); + let p = crate::kernels::quant::format::pack(fmt, &w, n, k); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, n, k); let x_f: Vec = (0..m * k).map(|i| ((i % 11) as f32 - 5.0) * 0.02).collect(); let x = unpack_f32(&pack_f32(&x_f, dt), dt); let mut expected = vec![0.0f32; m * n]; @@ -241,8 +241,8 @@ pub mod kernel_tests { // `== 4` branch, no regression). let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -406,7 +406,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn nax_bench( kernel: Kernel, @@ -423,11 +423,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n * k, DType::U8) } else { - (crate::quant::format::bitstream_words(n * k, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n * k, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = n * (k / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/gemm/patch_embed_block_scaled.rs b/crates/metaltile-std/src/kernels/gemm/patch_embed_block_scaled.rs index 55e7cd4e..9fd2b4c1 100644 --- a/crates/metaltile-std/src/kernels/gemm/patch_embed_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/gemm/patch_embed_block_scaled.rs @@ -13,7 +13,7 @@ //! `patch·hidden + h`). The per-`col` weight decode reuses the DSL decode //! intrinsics; `patch_dim` is a multiple of `block_size` (4-bit `block_size` a //! multiple of 8). fp8_e4m3 reuses the nvfp8 kernel. Codegen-only; correctness -//! pinned by the in-source `#[test_kernel]`s vs a `quant::format::dequant` oracle. +//! pinned by the in-source `#[test_kernel]`s vs a `kernels::quant::format::dequant` oracle. use metaltile::kernel; @@ -146,7 +146,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -173,8 +173,8 @@ pub mod kernel_tests { let bias_f = ramp(hidden, 5, 2.0); // Quantize the [hidden, patch_dim] projection weight via the shared codec. let w_f = ramp(hidden * patch_dim, 11, 4.0); - let p = crate::quant::format::pack(fmt, &w_f, hidden, patch_dim); - let wdq = crate::quant::format::dequant(fmt, &p, hidden, patch_dim); + let p = crate::kernels::quant::format::pack(fmt, &w_f, hidden, patch_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, hidden, patch_dim); let image = unpack_f32(&pack_f32(&image_f, dt), dt); let bias = unpack_f32(&pack_f32(&bias_f, dt), dt); // Oracle: explicit unfold + projection over the dequantized weight. @@ -205,8 +205,8 @@ pub mod kernel_tests { // buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -601,7 +601,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn patch_bench( @@ -624,11 +624,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n_codes, DType::U8) } else { - (crate::quant::format::bitstream_words(n_codes, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(n_codes, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = hidden * (patch_dim / fmt.block_size()); diff --git a/crates/metaltile-std/src/kernels/gemm/patch_embed_mma_block_scaled.rs b/crates/metaltile-std/src/kernels/gemm/patch_embed_mma_block_scaled.rs index 1d3a285d..24e20e1b 100644 --- a/crates/metaltile-std/src/kernels/gemm/patch_embed_mma_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/gemm/patch_embed_mma_block_scaled.rs @@ -33,7 +33,7 @@ //! - `B` is the quantized projection weight laid out as the 2-D matrix //! `[hidden, patch_dim]` (row `h` = hidden unit, column `kt = ic*patch_h*patch_w //! + py*patch_w + px`). This is the same row-major `[N, K]` weight the -//! `quant::format` packer produces. +//! `kernels::quant::format` packer produces. //! //! ## Quantized B-load (the only change vs dense patch_embed_mma) //! @@ -49,7 +49,7 @@ //! (1 byte/tap). The byte for tap `kt` is `h*patch_dim + kt`. //! //! - sub-byte symmetric int (int2/3/4/5/6 + MXINT2..6): weight is a FLAT -//! row-major u32 bit-stream, tight-packed LSB-first by `quant::format::pack`. +//! row-major u32 bit-stream, tight-packed LSB-first by `kernels::quant::format::pack`. //! The N-bit two's-complement code for tap `kt` of row `h` lives at GLOBAL //! bit offset `(h*patch_dim + kt)*N`, read straddle-aware across two words //! and float-sign-extended (`code - 2^N` when the top bit is set). `patch_dim` @@ -359,7 +359,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -444,8 +444,8 @@ pub mod kernel_tests { let image_f = ramp(in_ch * in_h * in_w, 13, 2.0); // Quantize the [hidden, patch_dim] projection weight via the shared codec. let weight_f = ramp(hidden * patch_dim, 11, 2.0); - let p = crate::quant::format::pack(fmt, &weight_f, hidden, patch_dim); - let wdq = crate::quant::format::dequant(fmt, &p, hidden, patch_dim); + let p = crate::kernels::quant::format::pack(fmt, &weight_f, hidden, patch_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, hidden, patch_dim); let bias_f = ramp(hidden, 5, 1.0); let image = unpack_f32(&pack_f32(&image_f, dt), dt); let bias = unpack_f32(&pack_f32(&bias_f, dt), dt); @@ -461,8 +461,8 @@ pub mod kernel_tests { // FP32 scales bind as f32; FP16 scales as f16; E8M0/E4M3 scales as one // byte. Driven off the format so each new precision picks the right type. let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -937,7 +937,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn mma_bench( @@ -962,11 +962,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n_weight, DType::U8) } else { - (crate::quant::format::bitstream_words(n_weight, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(n_weight, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = BenchSetup::new(kernel) diff --git a/crates/metaltile-std/src/ffai/dsv4_mhc.rs b/crates/metaltile-std/src/kernels/hyper_connections/mhc.rs similarity index 95% rename from crates/metaltile-std/src/ffai/dsv4_mhc.rs rename to crates/metaltile-std/src/kernels/hyper_connections/mhc.rs index 1a21adc1..a09e9069 100644 --- a/crates/metaltile-std/src/ffai/dsv4_mhc.rs +++ b/crates/metaltile-std/src/kernels/hyper_connections/mhc.rs @@ -16,7 +16,7 @@ //! ``` //! //! `pre`, `post`, and `comb` are DYNAMIC per-token tensors produced -//! by [`crate::ffai::dsv4_mhc_sinkhorn_split`] from the `hc_*_fn @ +//! by [`crate::kernels::hyper_connections::mhc_sinkhorn_split`] from the `hc_*_fn @ //! flatten(H)` 24-mix output. They are NOT stored model weights. //! //! ## Dispatch @@ -30,7 +30,7 @@ use metaltile::kernel; /// mHC collapse — `x[d, t] = sum_c pre[t, c] * H[d, c, t]`. #[kernel] -pub fn ffai_dsv4_mhc_collapse( +pub fn mt_mhc_collapse( state: Tensor, pre: Tensor, mut out: Tensor, @@ -63,7 +63,7 @@ pub fn ffai_dsv4_mhc_collapse( /// allocated output buffer in `state`. `comb` layout matches the /// split kernel output: `comb[t * n_hc * n_hc + dst * n_hc + src]`. #[kernel] -pub fn ffai_dsv4_mhc_expand( +pub fn mt_mhc_expand( block_out: Tensor, post: Tensor, comb: Tensor, @@ -98,7 +98,7 @@ pub fn ffai_dsv4_mhc_expand( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::{ffai_dsv4_mhc_collapse, ffai_dsv4_mhc_expand}; + use super::{mt_mhc_collapse, mt_mhc_expand}; use crate::utils::{pack_f32, unpack_f32}; fn cpu_collapse( @@ -128,7 +128,7 @@ pub mod kernel_tests { let pre: Vec = (0..n_tokens * n_hc).map(|i| (i as f32 - 4.0) * 0.2 + 0.5).collect(); let state_dt = unpack_f32(&pack_f32(&state, dt), dt); let expected = cpu_collapse(&state_dt, &pre, hidden_dim, n_hc, n_tokens); - TestSetup::new(ffai_dsv4_mhc_collapse::kernel_ir_for(dt)) + TestSetup::new(mt_mhc_collapse::kernel_ir_for(dt)) .input(TestBuffer::from_vec("state", pack_f32(&state, dt), dt)) .input(TestBuffer::from_vec("pre", pack_f32(&pre, DType::F32), DType::F32)) .input(TestBuffer::zeros("out", n_tokens * hidden_dim, dt)) @@ -184,7 +184,7 @@ pub mod kernel_tests { let residual_dt = unpack_f32(&pack_f32(&residual, dt), dt); let expected = cpu_expand(&block_out_dt, &post, &comb, &residual_dt, hidden_dim, n_hc, n_tokens); - TestSetup::new(ffai_dsv4_mhc_expand::kernel_ir_for(dt)) + TestSetup::new(mt_mhc_expand::kernel_ir_for(dt)) .input(TestBuffer::from_vec("block_out", pack_f32(&block_out, dt), dt)) .input(TestBuffer::from_vec("post", pack_f32(&post, DType::F32), DType::F32)) .input(TestBuffer::from_vec("comb", pack_f32(&comb, DType::F32), DType::F32)) @@ -207,12 +207,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::{ffai_dsv4_mhc_collapse, ffai_dsv4_mhc_expand}; + use super::{mt_mhc_collapse, mt_mhc_expand}; #[bench(dtypes = [f32, f16, bf16])] fn bench_collapse(dt: DType) -> BenchSetup { let (hidden_dim, n_hc, n_tokens) = (4096usize, 4usize, 1usize); - BenchSetup::new(ffai_dsv4_mhc_collapse::kernel_ir_for(dt)) + BenchSetup::new(mt_mhc_collapse::kernel_ir_for(dt)) .buffer(BenchBuffer::random("state", n_tokens * n_hc * hidden_dim, dt)) .buffer(BenchBuffer::random("pre", n_tokens * n_hc, DType::F32)) .buffer(BenchBuffer::zeros("out", n_tokens * hidden_dim, dt).output()) @@ -229,7 +229,7 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16])] fn bench_expand(dt: DType) -> BenchSetup { let (hidden_dim, n_hc, n_tokens) = (4096usize, 4usize, 1usize); - BenchSetup::new(ffai_dsv4_mhc_expand::kernel_ir_for(dt)) + BenchSetup::new(mt_mhc_expand::kernel_ir_for(dt)) .buffer(BenchBuffer::random("block_out", n_tokens * hidden_dim, dt)) .buffer(BenchBuffer::random("post", n_tokens * n_hc, DType::F32)) .buffer(BenchBuffer::random("comb", n_tokens * n_hc * n_hc, DType::F32)) diff --git a/crates/metaltile-std/src/ffai/dsv4_mhc_sinkhorn_split.rs b/crates/metaltile-std/src/kernels/hyper_connections/mhc_sinkhorn_split.rs similarity index 98% rename from crates/metaltile-std/src/ffai/dsv4_mhc_sinkhorn_split.rs rename to crates/metaltile-std/src/kernels/hyper_connections/mhc_sinkhorn_split.rs index 21a05f33..ec33b9e5 100644 --- a/crates/metaltile-std/src/ffai/dsv4_mhc_sinkhorn_split.rs +++ b/crates/metaltile-std/src/kernels/hyper_connections/mhc_sinkhorn_split.rs @@ -31,7 +31,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_dsv4_mhc_sinkhorn_split( +pub fn mt_mhc_sinkhorn_split( mixes: Tensor, scale: Tensor, base: Tensor, @@ -220,7 +220,7 @@ pub fn ffai_dsv4_mhc_sinkhorn_split( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_mhc_sinkhorn_split; + use super::mt_mhc_sinkhorn_split; use crate::utils::{pack_f32, unpack_f32}; #[allow(clippy::too_many_arguments)] @@ -300,7 +300,7 @@ pub mod kernel_tests { let eps = 1e-6_f32; let (pre_exp, post_exp, comb_exp) = cpu_reference(&mixes_dt, &scale, &base, n_tokens, eps, iters); - TestSetup::new(ffai_dsv4_mhc_sinkhorn_split::kernel_ir_for(dt)) + TestSetup::new(mt_mhc_sinkhorn_split::kernel_ir_for(dt)) .input(TestBuffer::from_vec("mixes", pack_f32(&mixes, dt), dt)) .input(TestBuffer::from_vec("scale", pack_f32(&scale, DType::F32), DType::F32)) .input(TestBuffer::from_vec("base", pack_f32(&base, DType::F32), DType::F32)) @@ -328,7 +328,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_mhc_sinkhorn_split; + use super::mt_mhc_sinkhorn_split; #[bench(dtypes = [f32, f16, bf16])] fn bench_split(dt: DType) -> BenchSetup { @@ -337,7 +337,7 @@ pub mod kernel_benches { // shape covers a prefill chunk of 256 tokens. let n_tokens = 256usize; let iters = 1usize; - BenchSetup::new(ffai_dsv4_mhc_sinkhorn_split::kernel_ir_for(dt)) + BenchSetup::new(mt_mhc_sinkhorn_split::kernel_ir_for(dt)) .buffer(BenchBuffer::random("mixes", n_tokens * 24, dt)) .buffer(BenchBuffer::random("scale", 3, DType::F32)) .buffer(BenchBuffer::random("base", 24, DType::F32)) diff --git a/crates/metaltile-std/src/kernels/hyper_connections/mod.rs b/crates/metaltile-std/src/kernels/hyper_connections/mod.rs new file mode 100644 index 00000000..1087ebf9 --- /dev/null +++ b/crates/metaltile-std/src/kernels/hyper_connections/mod.rs @@ -0,0 +1,8 @@ +//! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric +//! SPDX-License-Identifier: Apache-2.0 +//! Hyper-connection kernels — manifold-constrained hyper-connections (mHC): +//! dynamic residual mixing (collapse / expand) and the Sinkhorn dynamic-mix +//! split. A distinct architectural mechanism, not attention or normalization. + +pub mod mhc; +pub mod mhc_sinkhorn_split; diff --git a/crates/metaltile-std/src/kernels/mod.rs b/crates/metaltile-std/src/kernels/mod.rs index d6f116da..310345ab 100644 --- a/crates/metaltile-std/src/kernels/mod.rs +++ b/crates/metaltile-std/src/kernels/mod.rs @@ -10,12 +10,15 @@ pub mod audio; pub mod convolution; pub mod gemm; +pub mod hyper_connections; pub mod kv_cache; pub mod moe; pub mod norm; pub mod ops; pub mod primitives; +pub mod quant; pub mod rope; pub mod sampling; +pub mod sdpa; pub mod ssm; pub mod vision; diff --git a/crates/metaltile-std/src/kernels/moe/block_scaled_moe.rs b/crates/metaltile-std/src/kernels/moe/block_scaled_moe.rs index e5e44a9d..ff736915 100644 --- a/crates/metaltile-std/src/kernels/moe/block_scaled_moe.rs +++ b/crates/metaltile-std/src/kernels/moe/block_scaled_moe.rs @@ -165,7 +165,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{block_scaled_weights, pack_f32, unpack_f32}, }; @@ -217,8 +217,8 @@ pub mod kernel_tests { // correct for every sub-byte width. `in_dim` is a multiple of 32, so each // row's bit-stream is word-aligned for every width. let w = block_scaled_weights(stack_rows, in_dim); - let p = crate::quant::format::pack(fmt, &w, stack_rows, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, stack_rows, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, stack_rows, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, stack_rows, in_dim); // Deterministic per-token expert routing. let eids: Vec = (0..m_rows).map(|m| (m * 2 + 1) as u32 % n_experts as u32).collect(); let x_f: Vec = (0..m_rows * in_dim).map(|i| ((i % 11) as f32 - 5.0) * 0.01).collect(); @@ -232,8 +232,8 @@ pub mod kernel_tests { // pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -450,7 +450,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; /// Experts in the packed stack; throughput is independent of the count, but /// it sizes the weight buffer realistically. @@ -477,11 +477,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (stack_n, DType::U8) } else { - (crate::quant::format::bitstream_words(stack_n, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(stack_n, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); @@ -493,7 +496,7 @@ pub mod kernel_benches { let tok_codes = if fmt.element_bits() == 8 { out_dim * in_dim } else { - crate::quant::format::bitstream_words(out_dim * in_dim, fmt.element_bits()) + crate::kernels::quant::format::bitstream_words(out_dim * in_dim, fmt.element_bits()) }; let tok_blocks = out_dim * (in_dim / fmt.block_size()); let bytes = tok_codes * codes_dt.size_bytes() diff --git a/crates/metaltile-std/src/kernels/moe/dequant_gemv_expert_indexed_block_scaled.rs b/crates/metaltile-std/src/kernels/moe/dequant_gemv_expert_indexed_block_scaled.rs index b40b326d..b04583d2 100644 --- a/crates/metaltile-std/src/kernels/moe/dequant_gemv_expert_indexed_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/moe/dequant_gemv_expert_indexed_block_scaled.rs @@ -184,7 +184,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -235,11 +235,11 @@ pub mod kernel_tests { for e in 0..n_experts { stacked_w.extend_from_slice(&weights(e, out_dim, in_dim)); } - let p = crate::quant::format::pack(fmt, &stacked_w, stacked_rows, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &stacked_w, stacked_rows, in_dim); let sel_global = p.global; // Dequant the full stack, then slice the selected expert's row band for // the oracle (rows `[expert·out_dim, (expert+1)·out_dim)`). - let wdq_all = crate::quant::format::dequant(fmt, &p, stacked_rows, in_dim); + let wdq_all = crate::kernels::quant::format::dequant(fmt, &p, stacked_rows, in_dim); let wdq = &wdq_all[expert * out_dim * in_dim..(expert + 1) * out_dim * in_dim]; let input_f: Vec = (0..in_dim).map(|i| ((i % 11) as f32 - 5.0) * 0.01).collect(); @@ -253,8 +253,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -673,7 +673,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; #[allow(clippy::too_many_arguments)] fn expert_bench( @@ -695,16 +695,19 @@ pub mod kernel_benches { let total_codes = if fmt.element_bits() == 8 { n_experts * out_dim * in_dim } else { - crate::quant::format::bitstream_words(n_experts * out_dim * in_dim, fmt.element_bits()) + crate::kernels::quant::format::bitstream_words( + n_experts * out_dim * in_dim, + fmt.element_bits(), + ) }; let codes_per_expert = if fmt.element_bits() == 8 { out_dim * in_dim } else { - crate::quant::format::bitstream_words(out_dim * in_dim, fmt.element_bits()) + crate::kernels::quant::format::bitstream_words(out_dim * in_dim, fmt.element_bits()) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/kernels/moe/moe_mpp_block_scaled.rs b/crates/metaltile-std/src/kernels/moe/moe_mpp_block_scaled.rs index 1d3e361e..ad49ff84 100644 --- a/crates/metaltile-std/src/kernels/moe/moe_mpp_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/moe/moe_mpp_block_scaled.rs @@ -295,7 +295,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -351,9 +351,9 @@ pub mod kernel_tests { if i % 3 == 0 { -mag } else { mag } }) .collect(); - let p = crate::quant::format::pack(fmt, &stacked, stack_rows, k_in); + let p = crate::kernels::quant::format::pack(fmt, &stacked, stack_rows, k_in); let global = p.global; - let wdq = crate::quant::format::dequant(fmt, &p, stack_rows, k_in); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, stack_rows, k_in); // Activations: dtype-rounded so the GPU sees exactly the oracle's x. let x_f: Vec = (0..m_total * k_in).map(|i| ((i % 11) as f32 - 5.0) * 0.02).collect(); @@ -378,8 +378,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; @@ -704,7 +704,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; struct BlockBenchShape { n_experts: usize, @@ -728,11 +728,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (stack_n, DType::U8) } else { - (crate::quant::format::bitstream_words(stack_n, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(stack_n, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = n_experts * n_out * groups_per_row; diff --git a/crates/metaltile-std/src/kernels/moe/moe_mpp_bm64_block_scaled.rs b/crates/metaltile-std/src/kernels/moe/moe_mpp_bm64_block_scaled.rs index f6c5f656..83ec3c02 100644 --- a/crates/metaltile-std/src/kernels/moe/moe_mpp_bm64_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/moe/moe_mpp_bm64_block_scaled.rs @@ -306,7 +306,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -324,7 +324,7 @@ pub mod kernel_tests { /// `int8_indexed_setup`: per-row expert routing, dtype-rounded x, oracle = /// `Σ_k x[t,k] · dequant(W)[expert·n_out + nc, k]`. The whole stacked /// `[n_experts·n_out, k_in]` weight is packed in ONE call via - /// `quant::format::pack` (single contiguous bit-stream with one trailing + /// `kernels::quant::format::pack` (single contiguous bit-stream with one trailing /// guard word — never per-expert concat, which would misalign sub-byte /// widths) and the codes/scales are bound directly. No biases buffer; scale /// dtype + weight dtype are axis-driven off the format (`element_bits` / @@ -366,10 +366,10 @@ pub mod kernel_tests { if i % 3 == 0 { -mag } else { mag } }) .collect(); - let p = crate::quant::format::pack(fmt, &stacked, stack_rows, k_in); + let p = crate::kernels::quant::format::pack(fmt, &stacked, stack_rows, k_in); let global = p.global; // Dequant the whole stack once; the oracle slices per expert below. - let wdq = crate::quant::format::dequant(fmt, &p, stack_rows, k_in); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, stack_rows, k_in); // Activations: dtype-rounded so the GPU sees exactly the oracle's x. let x_f: Vec = (0..m_total * k_in).map(|i| ((i % 11) as f32 - 5.0) * 0.02).collect(); @@ -395,8 +395,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; @@ -719,7 +719,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; struct BlockBenchShape { n_experts: usize, @@ -743,11 +743,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (stack_n, DType::U8) } else { - (crate::quant::format::bitstream_words(stack_n, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(stack_n, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = n_experts * n_out * groups_per_row; diff --git a/crates/metaltile-std/src/kernels/moe/moe_mpp_bm8_block_scaled.rs b/crates/metaltile-std/src/kernels/moe/moe_mpp_bm8_block_scaled.rs index 7a46adee..898a4411 100644 --- a/crates/metaltile-std/src/kernels/moe/moe_mpp_bm8_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/moe/moe_mpp_bm8_block_scaled.rs @@ -298,7 +298,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -316,7 +316,7 @@ pub mod kernel_tests { /// Mirrors `int8_indexed_setup`: per-row expert routing, dtype-rounded x, /// oracle = `Σ_k x[t,k] · dequant(W_expert)[nc,k]`. Differs in that the /// whole `[n_experts·n_out, k_in]` expert stack is packed in ONE - /// `quant::format::pack` call (no biases buffer; scale dtype per format; + /// `kernels::quant::format::pack` call (no biases buffer; scale dtype per format; /// weight dtype U32 for sub-byte bit-streams / U8 for 8-bit). `block_size` /// and the nvfp4 `global` constexpr come from the packed tensor. The BM=8 /// m-tile height drives `ceil(m_total/8)` m-tiles. @@ -356,11 +356,11 @@ pub mod kernel_tests { if i % 3 == 0 { -mag } else { mag } }) .collect(); - let p = crate::quant::format::pack(fmt, &stacked, stack_rows, k_in); + let p = crate::kernels::quant::format::pack(fmt, &stacked, stack_rows, k_in); let global = p.global; // Dequant the full stack once; row `expert·n_out + nc` is expert `e`'s // output row `nc`. - let wdq = crate::quant::format::dequant(fmt, &p, stack_rows, k_in); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, stack_rows, k_in); // Activations: dtype-rounded so the GPU sees exactly the oracle's x. let x_f: Vec = (0..m_total * k_in).map(|i| ((i % 11) as f32 - 5.0) * 0.02).collect(); @@ -386,8 +386,8 @@ pub mod kernel_tests { // formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; @@ -708,7 +708,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; struct BlockBenchShape { n_experts: usize, @@ -733,11 +733,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (stack_n, DType::U8) } else { - (crate::quant::format::bitstream_words(stack_n, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(stack_n, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let n_blocks = n_experts * n_out * groups_per_row; diff --git a/crates/metaltile-std/src/kernels/norm/gated_rms_norm_block_scaled_qgemv.rs b/crates/metaltile-std/src/kernels/norm/gated_rms_norm_block_scaled_qgemv.rs index e6c86198..9c90bc65 100644 --- a/crates/metaltile-std/src/kernels/norm/gated_rms_norm_block_scaled_qgemv.rs +++ b/crates/metaltile-std/src/kernels/norm/gated_rms_norm_block_scaled_qgemv.rs @@ -204,7 +204,7 @@ pub mod kernel_tests { use metaltile::{core::ir::Kernel, test::*, test_kernel}; use super::*; - use crate::{quant::format::QFormat, utils::pack_f32}; + use crate::{kernels::quant::format::QFormat, utils::pack_f32}; const TPG: u32 = 64; const EPS: f32 = 1e-5; @@ -258,8 +258,8 @@ pub mod kernel_tests { if (i % 3) == 0 { -mag } else { mag } }) .collect(); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); // y fp32; z / norm_weight rounded through dt (kernel loads them as T). let y = source(in_dim, 0xA1, 2.0, 0.1); let z = round(&source(in_dim, 0xD4, 1.5, 0.0), dt); @@ -274,8 +274,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -590,7 +590,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn gated_bench( kernel: Kernel, @@ -608,11 +608,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n, DType::U8) } else { - (crate::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/kernels/norm/rms_norm_block_scaled_qgemv.rs b/crates/metaltile-std/src/kernels/norm/rms_norm_block_scaled_qgemv.rs index 770d7c81..6ddf36ee 100644 --- a/crates/metaltile-std/src/kernels/norm/rms_norm_block_scaled_qgemv.rs +++ b/crates/metaltile-std/src/kernels/norm/rms_norm_block_scaled_qgemv.rs @@ -188,7 +188,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{block_scaled_weights, pack_f32, unpack_f32}, }; @@ -220,8 +220,8 @@ pub mod kernel_tests { dt: DType, ) -> TestSetup { let w = block_scaled_weights(out_dim, in_dim); - let p = crate::quant::format::pack(fmt, &w, out_dim, in_dim); - let wdq = crate::quant::format::dequant(fmt, &p, out_dim, in_dim); + let p = crate::kernels::quant::format::pack(fmt, &w, out_dim, in_dim); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, out_dim, in_dim); // Round x / norm_weight through `dt` so the oracle sees what the GPU sees. let x_f: Vec = (0..in_dim).map(|i| ((i % 11) as f32 - 5.0) * 0.05 + 0.1).collect(); let nw_f: Vec = (0..in_dim).map(|i| 0.5 + (i % 7) as f32 * 0.1).collect(); @@ -235,8 +235,8 @@ pub mod kernel_tests { // pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -439,7 +439,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn rms_qgemv_bench( kernel: Kernel, @@ -455,11 +455,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n, DType::U8) } else { - (crate::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/ffai/gate_up_swiglu_fused.rs b/crates/metaltile-std/src/kernels/ops/gate_up_swiglu_fused.rs similarity index 93% rename from crates/metaltile-std/src/ffai/gate_up_swiglu_fused.rs rename to crates/metaltile-std/src/kernels/ops/gate_up_swiglu_fused.rs index b117ffb7..873c0ab7 100644 --- a/crates/metaltile-std/src/ffai/gate_up_swiglu_fused.rs +++ b/crates/metaltile-std/src/kernels/ops/gate_up_swiglu_fused.rs @@ -16,7 +16,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_gate_up_swiglu_fused( +pub fn mt_gate_up_swiglu_fused( gate_w: Tensor, up_w: Tensor, x: Tensor, @@ -45,7 +45,7 @@ pub fn ffai_gate_up_swiglu_fused( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_gate_up_swiglu_fused; + use super::mt_gate_up_swiglu_fused; use crate::utils::{pack_f32, unpack_f32}; fn setup(m: usize, k: usize, dt: DType) -> TestSetup { @@ -63,7 +63,7 @@ pub mod kernel_tests { sig * u_val }) .collect(); - TestSetup::new(ffai_gate_up_swiglu_fused::kernel_ir_for(dt)) + TestSetup::new(mt_gate_up_swiglu_fused::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("gate_w", pack_f32(&gate, dt), dt)) .input(TestBuffer::from_vec("up_w", pack_f32(&up, dt), dt)) @@ -81,12 +81,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gate_up_swiglu_fused; + use super::mt_gate_up_swiglu_fused; #[bench(dtypes = [f32, f16, bf16])] fn bench_gus(dt: DType) -> BenchSetup { let (m, k) = (2048usize, 4096usize); - BenchSetup::new(ffai_gate_up_swiglu_fused::kernel_ir_for(dt)) + BenchSetup::new(mt_gate_up_swiglu_fused::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("gate_w", m * k, dt)) .buffer(BenchBuffer::random("up_w", m * k, dt)) diff --git a/crates/metaltile-std/src/ffai/gelu_erf.rs b/crates/metaltile-std/src/kernels/ops/gelu_erf.rs similarity index 93% rename from crates/metaltile-std/src/ffai/gelu_erf.rs rename to crates/metaltile-std/src/kernels/ops/gelu_erf.rs index db8e5666..45716ab2 100644 --- a/crates/metaltile-std/src/ffai/gelu_erf.rs +++ b/crates/metaltile-std/src/kernels/ops/gelu_erf.rs @@ -18,7 +18,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_gelu_erf(input: Tensor, out: Tensor) { +pub fn mt_gelu_erf(input: Tensor, out: Tensor) { let i = program_id::<0>(); let x = load(input[i]).cast::(); // 0.5·x·(1 + erf(x · (1/√2))). @@ -29,7 +29,7 @@ pub fn ffai_gelu_erf(input: Tensor, out: Tensor) { pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_gelu_erf; + use super::mt_gelu_erf; use crate::utils::{pack_f32, unpack_f32}; // erf via the Abramowitz-Stegun 7.1.26 approximation — only the test @@ -57,7 +57,7 @@ pub mod kernel_tests { .iter() .map(|&x| 0.5 * x * (1.0 + erf_approx(x * std::f32::consts::FRAC_1_SQRT_2))) .collect(); - TestSetup::new(ffai_gelu_erf::kernel_ir_for(dt)) + TestSetup::new(mt_gelu_erf::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("input", pack_f32(&input_f, dt), dt)) .input(TestBuffer::zeros("out", n, dt)) @@ -70,12 +70,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gelu_erf; + use super::mt_gelu_erf; #[bench(dtypes = [f32, f16, bf16])] fn bench_gelu_erf(dt: DType) -> BenchSetup { let n = 2048usize * 64usize; - BenchSetup::new(ffai_gelu_erf::kernel_ir_for(dt)) + BenchSetup::new(mt_gelu_erf::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .buffer(BenchBuffer::random("input", n, dt)) .buffer(BenchBuffer::zeros("out", n, dt).output()) diff --git a/crates/metaltile-std/src/ffai/leaky_relu.rs b/crates/metaltile-std/src/kernels/ops/leaky_relu.rs similarity index 90% rename from crates/metaltile-std/src/ffai/leaky_relu.rs rename to crates/metaltile-std/src/kernels/ops/leaky_relu.rs index 1d4f0c72..6444a87d 100644 --- a/crates/metaltile-std/src/ffai/leaky_relu.rs +++ b/crates/metaltile-std/src/kernels/ops/leaky_relu.rs @@ -23,7 +23,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_leaky_relu(input: Tensor, out: Tensor, slope: Tensor) { +pub fn mt_leaky_relu(input: Tensor, out: Tensor, slope: Tensor) { let i = program_id::<0>(); let s = load(slope[0]); let x = load(input[i]).cast::(); @@ -32,12 +32,12 @@ pub fn ffai_leaky_relu(input: Tensor, out: Tensor, slope: Tensor) store(out[i], y.cast::()); } -/// New-syntax correctness for `ffai_leaky_relu`. Grid3D, grid `[n,1,1]`, tpg +/// New-syntax correctness for `mt_leaky_relu`. Grid3D, grid `[n,1,1]`, tpg /// `[1,1,1]`. Oracle applies the leaky rectifier per element. pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_leaky_relu; + use super::mt_leaky_relu; use crate::utils::{pack_f32, unpack_f32}; fn f32_bytes(v: f32) -> Vec { v.to_le_bytes().to_vec() } @@ -50,7 +50,7 @@ pub mod kernel_tests { let input_f: Vec = (0..n).map(|i| (i as f32 - 128.0) * 0.1).collect(); let input = unpack_f32(&pack_f32(&input_f, dt), dt); let exp: Vec = input.iter().map(|&x| if x > 0.0 { x } else { slope * x }).collect(); - TestSetup::new(ffai_leaky_relu::kernel_ir_for(dt)) + TestSetup::new(mt_leaky_relu::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("input", pack_f32(&input_f, dt), dt)) .input(TestBuffer::from_vec("slope", f32_bytes(slope), DType::F32)) @@ -64,12 +64,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_leaky_relu; + use super::mt_leaky_relu; #[bench(dtypes = [f32, f16, bf16])] fn bench_leaky_relu(dt: DType) -> BenchSetup { let n = 128usize * 7801usize; - BenchSetup::new(ffai_leaky_relu::kernel_ir_for(dt)) + BenchSetup::new(mt_leaky_relu::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .buffer(BenchBuffer::random("input", n, dt)) .buffer(BenchBuffer::from_vec("slope", 0.1f32.to_le_bytes().to_vec(), DType::F32)) diff --git a/crates/metaltile-std/src/kernels/ops/mod.rs b/crates/metaltile-std/src/kernels/ops/mod.rs index b53b31d6..154723a8 100644 --- a/crates/metaltile-std/src/kernels/ops/mod.rs +++ b/crates/metaltile-std/src/kernels/ops/mod.rs @@ -20,18 +20,22 @@ pub mod copy; // `fence.rs` is intentionally NOT declared: device/system memory fences and // atomics have no `#[kernel]` DSL representation (the "1 out of scope" op in the // audit). The file is kept for the implementation notes in its header. +pub mod gate_up_swiglu_fused; pub mod gated_activation; pub mod gather; pub mod gather_axis; +pub mod gelu_erf; pub mod hadamard; pub mod hadamard_m; pub mod indexing; +pub mod leaky_relu; pub mod logsumexp; pub mod random; pub mod reduce; pub mod scan; pub mod scatter_axis; pub mod strided; +pub mod swiglu_limit; pub mod ternary; pub mod unary; pub mod vector_add; diff --git a/crates/metaltile-std/src/ffai/dsv4_swiglu_limit.rs b/crates/metaltile-std/src/kernels/ops/swiglu_limit.rs similarity index 90% rename from crates/metaltile-std/src/ffai/dsv4_swiglu_limit.rs rename to crates/metaltile-std/src/kernels/ops/swiglu_limit.rs index d79cee59..d32fbed2 100644 --- a/crates/metaltile-std/src/ffai/dsv4_swiglu_limit.rs +++ b/crates/metaltile-std/src/kernels/ops/swiglu_limit.rs @@ -12,12 +12,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_dsv4_swiglu_limit( - gate: Tensor, - up: Tensor, - out: Tensor, - #[constexpr] limit: f32, -) { +pub fn mt_swiglu_limit(gate: Tensor, up: Tensor, out: Tensor, #[constexpr] limit: f32) { let idx = tid; let g = load(gate[idx]).cast::(); let u = load(up[idx]).cast::(); @@ -34,7 +29,7 @@ pub fn ffai_dsv4_swiglu_limit( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_swiglu_limit; + use super::mt_swiglu_limit; use crate::utils::{pack_f32, unpack_f32}; fn setup(n: usize, dt: DType) -> TestSetup { @@ -54,7 +49,7 @@ pub mod kernel_tests { s * ul }) .collect(); - TestSetup::new(ffai_dsv4_swiglu_limit::kernel_ir_for(dt)) + TestSetup::new(mt_swiglu_limit::kernel_ir_for(dt)) .input(TestBuffer::from_vec("gate", pack_f32(&gate, dt), dt)) .input(TestBuffer::from_vec("up", pack_f32(&up, dt), dt)) .input(TestBuffer::zeros("out", n, dt)) @@ -76,12 +71,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_swiglu_limit; + use super::mt_swiglu_limit; #[bench(dtypes = [f32, f16, bf16])] fn bench_dsv4_swiglu_limit(dt: DType) -> BenchSetup { let n = 1024 * 1024usize; - BenchSetup::new(ffai_dsv4_swiglu_limit::kernel_ir_for(dt)) + BenchSetup::new(mt_swiglu_limit::kernel_ir_for(dt)) .buffer(BenchBuffer::random("gate", n, dt)) .buffer(BenchBuffer::random("up", n, dt)) .buffer(BenchBuffer::zeros("out", n, dt).output()) diff --git a/crates/metaltile-std/src/kernels/primitives.rs b/crates/metaltile-std/src/kernels/primitives.rs index 6204edb2..33755d37 100644 --- a/crates/metaltile-std/src/kernels/primitives.rs +++ b/crates/metaltile-std/src/kernels/primitives.rs @@ -7,7 +7,7 @@ //! use the cross-kernel call syntax (`let v = mt_decode_e2m1(nib)`); //! `KernelInlinePass` splices the body inline at codegen, so there is no memory //! round-trip — exactly as fast as pasting the body, written once. They are the -//! kernel-side mirror of `quant::codec` (the Rust host-side source of truth); +//! kernel-side mirror of `kernels::quant::codec` (the Rust host-side source of truth); //! kernel and oracle decode through the same math so they cannot drift. //! //! Used by the `variants(FMT = […])` format-axis folds across the weight-bearing @@ -50,7 +50,7 @@ pub fn mt_decode_e2m1(inp: Tensor, out: Tensor) { /// Decode an 8-bit E4M3 (fp8) code → f32. /// /// Format: 1 sign · 4 exp (bias 7) · 3 mantissa. Max ±448, no infinity. -/// Mirrors `quant::codec::e4m3_decode`. +/// Mirrors `kernels::quant::codec::e4m3_decode`. #[kernel] pub fn mt_decode_e4m3(inp: Tensor, out: Tensor) { let c = load(inp[0u32]); @@ -65,7 +65,7 @@ pub fn mt_decode_e4m3(inp: Tensor, out: Tensor) { /// Decode an 8-bit E5M2 (fp8) code → f32. /// /// Format: 1 sign · 5 exp (bias 15) · 2 mantissa — the high byte of an IEEE -/// half. Mirrors `quant::codec::e5m2_decode`. +/// half. Mirrors `kernels::quant::codec::e5m2_decode`. #[kernel] pub fn mt_decode_e5m2(inp: Tensor, out: Tensor) { let c = load(inp[0u32]); @@ -80,7 +80,7 @@ pub fn mt_decode_e5m2(inp: Tensor, out: Tensor) { /// Decode a symmetric-int8 code → f32. /// /// Reinterprets the low 8 bits of `inp` as a signed byte (two's complement -/// sign-extension via `<< 24 / >> 24`). Mirrors `quant::codec::int8_decode`. +/// sign-extension via `<< 24 / >> 24`). Mirrors `kernels::quant::codec::int8_decode`. #[kernel] pub fn mt_decode_int8(inp: Tensor, out: Tensor) { let c = load(inp[0u32]); diff --git a/crates/metaltile-std/src/ffai/aura_dequant_rotated.rs b/crates/metaltile-std/src/kernels/quant/aura_dequant_rotated.rs similarity index 96% rename from crates/metaltile-std/src/ffai/aura_dequant_rotated.rs rename to crates/metaltile-std/src/kernels/quant/aura_dequant_rotated.rs index d902157c..f5af9fec 100644 --- a/crates/metaltile-std/src/ffai/aura_dequant_rotated.rs +++ b/crates/metaltile-std/src/kernels/quant/aura_dequant_rotated.rs @@ -43,7 +43,7 @@ use metaltile::kernel; /// AURA codebook dequant + norm-scale, Grid3D dispatch. /// -/// Produces kernels: `aura_dequant_rotated_int2`, `_int3`, `_int4`, +/// Produces kernels: `mt_aura_dequant_rotated_int2`, `_int3`, `_int4`, /// `_int6`, `_int8`. /// /// Even BITS (2, 4, 8): pack-strided — one u32 load amortises across all @@ -53,7 +53,7 @@ use metaltile::kernel; /// /// Grid: (packed_width, tokens, B*H), tpg=[1,1,1]. #[kernel(variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] -pub fn aura_dequant_rotated( +pub fn mt_aura_dequant_rotated( packed: Tensor, norms: Tensor, codebook: Tensor, @@ -112,7 +112,7 @@ pub fn aura_dequant_rotated( } } -/// Correctness tests for the `aura_dequant_rotated_int{2,3,4,6,8}` family. +/// Correctness tests for the `mt_aura_dequant_rotated_int{2,3,4,6,8}` family. /// Grid3D, one thread per (packed_word, token, bh) tile. Oracle: decode each /// codebook index from the packed bit-stream and multiply by the token norm. /// dim=128 is u32-aligned for every supported bit-width (128×2/32=8, @@ -195,7 +195,7 @@ pub mod kernel_tests { #[test_kernel(dtypes = [f32, f16, bf16], tol = 1e-1, variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] fn test_aura_dequant_rotated(dt: DType) -> TestSetup { - dequant_setup(aura_dequant_rotated_intBITS::kernel_ir_for(dt), BITS, 128, 2, 3, dt) + dequant_setup(mt_aura_dequant_rotated_intBITS::kernel_ir_for(dt), BITS, 128, 2, 3, dt) } } @@ -232,7 +232,7 @@ pub mod kernel_benches { variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] fn bench_aura_dequant_rotated(dt: DType) -> BenchSetup { setup( - BenchSetup::new(aura_dequant_rotated_intBITS::kernel_ir_for(dt)), + BenchSetup::new(mt_aura_dequant_rotated_intBITS::kernel_ir_for(dt)), 128, BITS, 8, diff --git a/crates/metaltile-std/src/ffai/aura_encode.rs b/crates/metaltile-std/src/kernels/quant/aura_encode.rs similarity index 97% rename from crates/metaltile-std/src/ffai/aura_encode.rs rename to crates/metaltile-std/src/kernels/quant/aura_encode.rs index 32035c58..e1ef26f1 100644 --- a/crates/metaltile-std/src/ffai/aura_encode.rs +++ b/crates/metaltile-std/src/kernels/quant/aura_encode.rs @@ -76,7 +76,7 @@ //! //! ## Macro structure //! -//! `aura_encode_kernel!` wraps a single `#[kernel] pub fn …` +//! `mt_aura_encode_kernel!` wraps a single `#[kernel] pub fn …` //! at module scope. Bit-widths get separate invocations so the compiler //! expands the outer macro before the `#[kernel]` proc-macro sees it — //! required because the proc-macro does not expand inner declarative macros. @@ -85,8 +85,8 @@ use metaltile::kernel; /// AURA fused rotation-encode kernel — variable bit-widths (2, 3, 4, 6, 8). /// -/// Produces kernels: `aura_encode_int2`, `aura_encode_int3`, `aura_encode_int4`, -/// `aura_encode_int6`, `aura_encode_int8`. +/// Produces kernels: `mt_aura_encode_int2`, `mt_aura_encode_int3`, `mt_aura_encode_int4`, +/// `mt_aura_encode_int6`, `mt_aura_encode_int8`. /// /// Each variant bakes in `BITS` (the code width) and `LEVELS = 2^BITS` (the /// number of codebook entries). Five fused stages per row: @@ -124,7 +124,7 @@ use metaltile::kernel; // → 1.40 (mirror baseline 1.41). The 0.3-nat gap maps cleanly // to the bf16 boundary rounding flipping borderline bins. #[kernel(variants(BITS = [2, 3, 4, 6, 8], LEVELS = [4, 8, 16, 64, 256], suffix = "int{BITS}"))] -pub fn aura_encode( +pub fn mt_aura_encode( input: Tensor, rotation: Tensor, boundaries: Tensor, @@ -211,7 +211,7 @@ pub fn aura_encode( // with garbage shifted by `masked >> 32` (which Apple // Silicon evaluates as `masked >> 0 = masked`). Symptom: // low nibbles of subsequent words OR'd with unrelated dim - // indices. Caught by the aura_encode GPU correctness + // indices. Caught by the mt_aura_encode GPU correctness // test. See FFAI post-mortem 2026-05-19 — a metaltile // codegen follow-up should emit i32 comparisons // faithfully when the DSL requests them. @@ -254,7 +254,7 @@ pub fn aura_encode( /// affine-quantize tests' strategy: the **packed codes** go through a /// branchless boundary-count whose last bit and bit-packing are sensitive to /// Metal fast-math FMA fusion, so they stayed covered by the legacy bit-exact -/// `tests/aura_encode_gpu_correctness.rs` (removed in #240) A/B test. Here we +/// `tests/mt_aura_encode_gpu_correctness.rs` (removed in #240) A/B test. Here we /// pin the part that is robustly checkable — the per-vector /// **norm-correction factor** (`norms_out`), which the decoder multiplies /// back through. @@ -276,7 +276,7 @@ pub fn aura_encode( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::aura_encode_int4; + use super::mt_aura_encode_int4; use crate::utils::{pack_f32, unpack_f32}; fn u32_bytes(v: &[u32]) -> Vec { v.iter().flat_map(|x| x.to_le_bytes()).collect() } @@ -398,7 +398,7 @@ pub mod kernel_tests { norms_oracle(&input_r, &rotation_r, &boundaries, &codebook_r, rows, dim); let expected_norms = unpack_f32(&pack_f32(&expected_norms_f32, dt), dt); - TestSetup::new(aura_encode_int4::kernel_ir_for(dt)) + TestSetup::new(mt_aura_encode_int4::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("input", pack_f32(input, dt), dt)) .input(TestBuffer::from_vec("rotation", pack_f32(rotation, dt), dt)) @@ -451,7 +451,7 @@ pub mod kernel_tests { // exercised against the oracle (which applies the same Π). f32-only and a // ramp input that keeps rotated values off the Lloyd-Max boundaries, so // the matmul-reorder noise can't flip a quant index (mirrors the legacy - // `aura_encode_int4_srht_rotation_f32` A/B test). + // `mt_aura_encode_int4_srht_rotation_f32` A/B test). #[test_kernel(dtypes = [f32], tol = 1e-4)] fn test_aura_encode_int4_srht_rotation(dt: DType) -> TestSetup { let dim = 128usize; @@ -488,6 +488,6 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16], variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] fn bench_aura_encode(dt: DType) -> BenchSetup { - setup(BenchSetup::new(aura_encode_intBITS::kernel_ir_for(dt)), 128, BITS, 256, dt) + setup(BenchSetup::new(mt_aura_encode_intBITS::kernel_ir_for(dt)), 128, BITS, 256, dt) } } diff --git a/crates/metaltile-std/src/mlx/block_scaled_dequant.rs b/crates/metaltile-std/src/kernels/quant/block_scaled_dequant.rs similarity index 97% rename from crates/metaltile-std/src/mlx/block_scaled_dequant.rs rename to crates/metaltile-std/src/kernels/quant/block_scaled_dequant.rs index 4188762b..a7fa5645 100644 --- a/crates/metaltile-std/src/mlx/block_scaled_dequant.rs +++ b/crates/metaltile-std/src/kernels/quant/block_scaled_dequant.rs @@ -5,8 +5,8 @@ //! //! Each kernel reads packed element codes + per-block scales and writes the //! reconstructed `[rows, cols]` matrix. The decode math mirrors -//! [`crate::quant::codec`] exactly, so the GPU output is checked against the -//! host [`crate::quant::format::dequant`] oracle — same reference, no drift. +//! [`crate::kernels::quant::codec`] exactly, so the GPU output is checked against the +//! host [`crate::kernels::quant::format::dequant`] oracle — same reference, no drift. //! //! ## DISPATCH INVARIANTS (all kernels here) //! @@ -399,7 +399,7 @@ pub mod kernel_tests { use metaltile::{core::ir::Kernel, test::*, test_kernel}; use super::*; - use crate::{quant::format::QFormat, utils::pack_f32}; + use crate::{kernels::quant::format::QFormat, utils::pack_f32}; /// Deterministic f32 weights with magnitude varying along K (so per-block /// scales differ) and mixed signs. @@ -416,7 +416,7 @@ pub mod kernel_tests { /// Shared setup: pack `[rows, cols]` weights in `fmt`, dispatch the dequant /// kernel, and expect the host oracle's reconstruction. Kernel and oracle - /// share `quant::codec`, so the match is near-exact. + /// share `kernels::quant::codec`, so the match is near-exact. fn dequant_setup( kernel: Kernel, fmt: QFormat, @@ -425,8 +425,8 @@ pub mod kernel_tests { dt: DType, ) -> TestSetup { let w = weights(rows, cols); - let p = crate::quant::format::pack(fmt, &w, rows, cols); - let oracle = crate::quant::format::dequant(fmt, &p, rows, cols); + let p = crate::kernels::quant::format::pack(fmt, &w, rows, cols); + let oracle = crate::kernels::quant::format::dequant(fmt, &p, rows, cols); let n = rows * cols; const TPG: u32 = 256; // Sub-byte codes (int2-6, E2M1) bind as bit-stream u32 words; 8-bit codes @@ -434,8 +434,8 @@ pub mod kernel_tests { // scales as one byte. let codes_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -606,7 +606,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn dequant_bench( kernel: Kernel, @@ -622,11 +622,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n, DType::U8) } else { - (crate::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let bytes = codes_len * codes_dt.size_bytes() diff --git a/crates/metaltile-std/src/quant/codec.rs b/crates/metaltile-std/src/kernels/quant/codec.rs similarity index 100% rename from crates/metaltile-std/src/quant/codec.rs rename to crates/metaltile-std/src/kernels/quant/codec.rs diff --git a/crates/metaltile-std/src/ffai/dequant_gather.rs b/crates/metaltile-std/src/kernels/quant/dequant_gather.rs similarity index 94% rename from crates/metaltile-std/src/ffai/dequant_gather.rs rename to crates/metaltile-std/src/kernels/quant/dequant_gather.rs index c69ffcdf..72571250 100644 --- a/crates/metaltile-std/src/ffai/dequant_gather.rs +++ b/crates/metaltile-std/src/kernels/quant/dequant_gather.rs @@ -32,7 +32,7 @@ //! //! ## Macro structure //! -//! `dequant_gather_kernel!` emits the entire `#[kernel] pub fn …` +//! `mt_dequant_gather_kernel!` emits the entire `#[kernel] pub fn …` //! at module scope. The compiler expands the outer macro before the //! `#[kernel]` proc-macro runs, so the body parser sees concrete tokens //! with `$bits` already substituted. Embedding the body inside an *inner* @@ -43,7 +43,7 @@ use metaltile::kernel; /// Dequantizing token-gather kernel — variable bit-widths (2, 3, 4, 5, 6, 8). /// -/// Produces kernels: `dequant_gather_int2`, `_int3`, `_int4`, `_int5`, +/// Produces kernels: `mt_dequant_gather_int2`, `_int3`, `_int4`, `_int5`, /// `_int6`, `_int8`. /// /// Uses a straddle-aware two-word bit-stream read that handles every width @@ -53,7 +53,7 @@ use metaltile::kernel; /// /// Grid: Elementwise, one thread per `(token, d)` output element. #[kernel(variants(BITS = [2, 3, 4, 5, 6, 8], suffix = "int{BITS}"))] -pub fn dequant_gather( +pub fn mt_dequant_gather( weight: Tensor, scales: Tensor, biases: Tensor, @@ -94,7 +94,7 @@ pub fn dequant_gather( store(out[idx], w_real.cast::()); } -/// New-syntax correctness tests for the `dequant_gather_int{2,3,4,5,6,8}` +/// New-syntax correctness tests for the `mt_dequant_gather_int{2,3,4,5,6,8}` /// quantized-embedding-gather family. Grid3D, one thread per output element /// (`n_tokens * hidden` threads via `grid_1d` ceil-div). /// @@ -142,7 +142,7 @@ pub mod kernel_tests { /// CPU oracle: per `(token, d)`, gather row `indices[token]`, unpack the /// `bits`-wide code at bit offset `d*bits`, dequantize via `q*scale+bias`. #[allow(clippy::too_many_arguments)] - fn dequant_gather_oracle( + fn mt_dequant_gather_oracle( weight: &[u32], scales: &[f32], biases: &[f32], @@ -200,7 +200,7 @@ pub mod kernel_tests { let n_tokens = indices.len(); let s = unpack_f32(&pack_f32(&scales_f, dt), dt); let b = unpack_f32(&pack_f32(&biases_f, dt), dt); - let expected = dequant_gather_oracle(&w, &s, &b, &indices, hidden, group_size, bits); + let expected = mt_dequant_gather_oracle(&w, &s, &b, &indices, hidden, group_size, bits); TestSetup::new(kernel) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("weight", u32_bytes(&w), DType::U32)) @@ -218,7 +218,7 @@ pub mod kernel_tests { #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-4, 1e-2, 1e-1], variants(BITS = [2, 4, 8], suffix = "int{BITS}"))] fn test_dequant_gather_pow2(dt: DType) -> TestSetup { - gather_setup(dequant_gather_intBITS::kernel_ir_for(dt), BITS, 256, 64, dt) + gather_setup(mt_dequant_gather_intBITS::kernel_ir_for(dt), BITS, 256, 64, dt) } // Odd widths — hidden*BITS must be a multiple of 32 (hidden=64, group_size=32). @@ -226,7 +226,7 @@ pub mod kernel_tests { #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-4, 1e-2, 1e-1], variants(BITS = [3, 5, 6], suffix = "int{BITS}"))] fn test_dequant_gather_odd(dt: DType) -> TestSetup { - gather_setup(dequant_gather_intBITS::kernel_ir_for(dt), BITS, 64, 32, dt) + gather_setup(mt_dequant_gather_intBITS::kernel_ir_for(dt), BITS, 64, 32, dt) } } @@ -260,6 +260,6 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16], variants(BITS = [2, 3, 4, 5, 6, 8], suffix = "int{BITS}"))] fn bench_dequant_gather(dt: DType) -> BenchSetup { - gb(dequant_gather_intBITS::kernel_ir_for(dt), BITS, 4096, 64, dt) + gb(mt_dequant_gather_intBITS::kernel_ir_for(dt), BITS, 4096, 64, dt) } } diff --git a/crates/metaltile-std/src/ffai/dequant_gather_block_scaled.rs b/crates/metaltile-std/src/kernels/quant/dequant_gather_block_scaled.rs similarity index 98% rename from crates/metaltile-std/src/ffai/dequant_gather_block_scaled.rs rename to crates/metaltile-std/src/kernels/quant/dequant_gather_block_scaled.rs index e14d70d2..87f50b22 100644 --- a/crates/metaltile-std/src/ffai/dequant_gather_block_scaled.rs +++ b/crates/metaltile-std/src/kernels/quant/dequant_gather_block_scaled.rs @@ -6,7 +6,7 @@ //! For each output element `(token, d)`: gather row `indices[token]` of a //! `[vocab, hidden]` block-scaled table, decode the element at column `d` //! (E2M1 nibble / E4M3 / E5M2 byte) × its block scale, store to -//! `out[token, d]`. The block-scaled counterpart of `ffai/dequant_gather.rs` +//! `out[token, d]`. The block-scaled counterpart of `ffai/mt_dequant_gather.rs` //! (which handles int2–int8 affine). Pure dequant — no reduction. //! //! ## DISPATCH INVARIANTS @@ -470,7 +470,7 @@ pub mod kernel_tests { use metaltile::{core::ir::Kernel, test::*, test_kernel}; use super::*; - use crate::{quant::format::QFormat, utils::pack_f32}; + use crate::{kernels::quant::format::QFormat, utils::pack_f32}; fn u32_bytes(v: &[u32]) -> Vec { v.iter().flat_map(|x| x.to_le_bytes()).collect() } @@ -489,8 +489,8 @@ pub mod kernel_tests { fn gather_setup(kernel: Kernel, fmt: QFormat, hidden: usize, dt: DType) -> TestSetup { let vocab = 8usize; let w = table(vocab, hidden); - let p = crate::quant::format::pack(fmt, &w, vocab, hidden); - let wdq = crate::quant::format::dequant(fmt, &p, vocab, hidden); + let p = crate::kernels::quant::format::pack(fmt, &w, vocab, hidden); + let wdq = crate::kernels::quant::format::dequant(fmt, &p, vocab, hidden); // Non-monotonic gather that repeats row 4 — surfaces token→row bugs. let indices: Vec = vec![3, 0, 7, 1, 4, 4]; let n_tokens = indices.len(); @@ -505,8 +505,8 @@ pub mod kernel_tests { // scales as half; E8M0 / E4M3 scales as one byte. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -671,7 +671,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn gb(kernel: Kernel, fmt: QFormat, hidden: usize, dt: DType) -> BenchSetup { let vocab = 4096usize; @@ -684,11 +684,11 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (n, DType::U8) } else { - (crate::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) + (crate::kernels::quant::format::bitstream_words(n, fmt.element_bits()), DType::U32) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = BenchSetup::new(kernel) diff --git a/crates/metaltile-std/src/ffai/ffai_dequant_q4.rs b/crates/metaltile-std/src/kernels/quant/dequant_q4.rs similarity index 96% rename from crates/metaltile-std/src/ffai/ffai_dequant_q4.rs rename to crates/metaltile-std/src/kernels/quant/dequant_q4.rs index b3dc9fc4..931ea4e9 100644 --- a/crates/metaltile-std/src/ffai/ffai_dequant_q4.rs +++ b/crates/metaltile-std/src/kernels/quant/dequant_q4.rs @@ -9,7 +9,7 @@ //! the right move is to dequant the weight ONCE to f16 and feed the //! tensor-core `mt_gemm`. This kernel is that one-time expansion. //! -//! ## Q4 block layout (matches `ffai_ops::quantize_q4`) +//! ## Q4 block layout (matches `mt_ops::quantize_q4`) //! //! Per row `r`, the `k` columns are grouped into `bpr = k/32` blocks of //! 32 values. Each block is 4 packed u32 words (`word` 0..3), each word @@ -52,7 +52,7 @@ use metaltile::kernel; // Scales are f16 (the resident projection/expert weights store amax/7 as f16 // via the loader's `tb_f16`). The kernel widens to f32 for the multiply. #[kernel] -pub fn ffai_dequant_q4( +pub fn mt_dequant_q4( qs: Tensor, scales: Tensor, out: Tensor, @@ -94,10 +94,10 @@ pub fn ffai_dequant_q4( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dequant_q4; + use super::mt_dequant_q4; use crate::utils::pack_f32; - /// Reference Q4 quantizer — mirrors `ffai_ops::quantize_q4` exactly: + /// Reference Q4 quantizer — mirrors `mt_ops::quantize_q4` exactly: /// signed 4-bit, per-32-block scale = amax/7, 4 u32 words/block. fn quantize_q4(w: &[f32], m: usize, k: usize) -> (Vec, Vec) { let bpr = k / 32; @@ -152,7 +152,7 @@ pub mod kernel_tests { scales.iter().map(|&s| half::f16::from_f32(s).to_f32()).collect(); let dequantized = cpu_dequant(&qs, &scales_f16, m, k); let qs_bytes: Vec = qs.iter().flat_map(|x| x.to_le_bytes()).collect(); - TestSetup::new(ffai_dequant_q4::kernel_ir_for(dt)) + TestSetup::new(mt_dequant_q4::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("qs", qs_bytes, DType::U32)) .input(TestBuffer::from_vec("scales", pack_f32(&scales, DType::F16), DType::F16)) diff --git a/crates/metaltile-std/src/quant/format.rs b/crates/metaltile-std/src/kernels/quant/format.rs similarity index 100% rename from crates/metaltile-std/src/quant/format.rs rename to crates/metaltile-std/src/kernels/quant/format.rs diff --git a/crates/metaltile-std/src/ffai/dsv4_fp8_block_dequant.rs b/crates/metaltile-std/src/kernels/quant/fp8_block_dequant.rs similarity index 95% rename from crates/metaltile-std/src/ffai/dsv4_fp8_block_dequant.rs rename to crates/metaltile-std/src/kernels/quant/fp8_block_dequant.rs index b10f69a1..c3ed2e36 100644 --- a/crates/metaltile-std/src/ffai/dsv4_fp8_block_dequant.rs +++ b/crates/metaltile-std/src/kernels/quant/fp8_block_dequant.rs @@ -41,7 +41,7 @@ //! Bench-stage matters: for matrix-vector decode (FFAI hot path), //! the dequant-then-matmul fallback (which this kernel is the //! dequant half of) ships fp16 weights to a downstream gemv. The -//! fused `ffai_dsv4_fp8_block_gemv` lands as a follow-up — Apple +//! fused `mt_fp8_block_gemv` lands as a follow-up — Apple //! has no native FP8 multiply, but a dequant-on-the-fly variant //! that interleaves the LUT lookup with the gemv accumulator can //! avoid materialising the full fp16 matrix. @@ -50,7 +50,7 @@ use metaltile::kernel; // Bare `#[kernel]` — mixed-dtype param set (concrete u8/f32 + generic T). #[kernel] -pub fn ffai_dsv4_fp8_block_dequant( +pub fn mt_fp8_block_dequant( weight_bytes: Tensor, scales: Tensor, fp8_lut: Tensor, @@ -84,12 +84,12 @@ pub fn ffai_dsv4_fp8_block_dequant( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_fp8_block_dequant; - use crate::{quant::codec, utils::pack_f32}; + use super::mt_fp8_block_dequant; + use crate::{kernels::quant::codec, utils::pack_f32}; /// FP8 e4m3 byte → fp32 LUT. The finite-value decode is the shared /// `codec::e4m3_decode` — the single source of truth every E4M3 path - /// (`quant::format`'s E4M3 element, the GGUF/MLX micro-float scales) routes + /// (`kernels::quant::format`'s E4M3 element, the GGUF/MLX micro-float scales) routes /// through — so this DSv4 model LUT can't drift from the kernel's decode. /// DSv4's FP8 storage additionally reserves the all-ones magnitude code /// (`0x7F` / `0xFF`) as a NaN sentinel; production weights never emit it @@ -176,7 +176,7 @@ pub mod kernel_tests { let (bytes, scales) = quantize_block_fp8(&values, m, n); let lut = build_fp8_lut(); let dequantized = cpu_dequant(&bytes, &scales, m, n); - TestSetup::new(ffai_dsv4_fp8_block_dequant::kernel_ir_for(dt)) + TestSetup::new(mt_fp8_block_dequant::kernel_ir_for(dt)) .input(TestBuffer::from_vec("weight_bytes", bytes, DType::U8)) .input(TestBuffer::from_vec("scales", pack_f32(&scales, DType::F32), DType::F32)) .input(TestBuffer::from_vec("fp8_lut", pack_f32(&lut, DType::F32), DType::F32)) @@ -200,7 +200,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_fp8_block_dequant; + use super::mt_fp8_block_dequant; #[bench(dtypes = [f32, f16, bf16])] fn bench_fp8(dt: DType) -> BenchSetup { @@ -210,7 +210,7 @@ pub mod kernel_benches { let (m, n) = (4096usize, 4096usize); let block_rows = m / 128; let block_cols = n / 128; - BenchSetup::new(ffai_dsv4_fp8_block_dequant::kernel_ir_for(dt)) + BenchSetup::new(mt_fp8_block_dequant::kernel_ir_for(dt)) .buffer(BenchBuffer::random("weight_bytes", m * n, DType::U8)) .buffer(BenchBuffer::random("scales", block_rows * block_cols, DType::F32)) .buffer(BenchBuffer::random("fp8_lut", 256, DType::F32)) diff --git a/crates/metaltile-std/src/quant/gguf.rs b/crates/metaltile-std/src/kernels/quant/gguf.rs similarity index 100% rename from crates/metaltile-std/src/quant/gguf.rs rename to crates/metaltile-std/src/kernels/quant/gguf.rs diff --git a/crates/metaltile-std/src/ffai/gguf_dequant_iq2_xxs.rs b/crates/metaltile-std/src/kernels/quant/gguf_dequant_iq2_xxs.rs similarity index 96% rename from crates/metaltile-std/src/ffai/gguf_dequant_iq2_xxs.rs rename to crates/metaltile-std/src/kernels/quant/gguf_dequant_iq2_xxs.rs index dfb92c62..99e4991d 100644 --- a/crates/metaltile-std/src/ffai/gguf_dequant_iq2_xxs.rs +++ b/crates/metaltile-std/src/kernels/quant/gguf_dequant_iq2_xxs.rs @@ -88,7 +88,7 @@ use metaltile::kernel; // Both are uploaded once at runtime init and shared across all // dequant calls. #[kernel] -pub fn ffai_gguf_dequant_iq2_xxs( +pub fn mt_gguf_dequant_iq2_xxs( qs_u32: Tensor, d_f32: Tensor, grid: Tensor, @@ -146,7 +146,7 @@ pub mod kernel_tests { use metaltile::test::*; - use super::ffai_gguf_dequant_iq2_xxs; + use super::mt_gguf_dequant_iq2_xxs; use crate::utils::pack_f32; /// MSL emission smoke test — confirms the kernel codegens for all @@ -156,7 +156,7 @@ pub mod kernel_tests { #[test] fn codegen_iq2_xxs_smoke() { for dt in [DType::F32, DType::F16, DType::BF16] { - let ir = ffai_gguf_dequant_iq2_xxs::kernel_ir_for(dt); + let ir = mt_gguf_dequant_iq2_xxs::kernel_ir_for(dt); assert!(!ir.body.ops.is_empty(), "kernel body emitted no ops for {dt:?}"); assert!(ir.params.iter().any(|p| p.name == "qs_u32"), "missing qs_u32 param"); assert!(ir.params.iter().any(|p| p.name == "grid"), "missing grid param"); @@ -171,7 +171,7 @@ pub mod kernel_tests { #[allow(dead_code)] fn _placeholder_setup(n_blocks: usize, dt: DType) -> TestSetup { let n = n_blocks * 256; - TestSetup::new(ffai_gguf_dequant_iq2_xxs::kernel_ir_for(dt)) + TestSetup::new(mt_gguf_dequant_iq2_xxs::kernel_ir_for(dt)) .input(TestBuffer::zeros("qs_u32", n_blocks * 16, DType::U32)) .input(TestBuffer::from_vec( "d_f32", @@ -190,13 +190,13 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gguf_dequant_iq2_xxs; + use super::mt_gguf_dequant_iq2_xxs; #[bench(dtypes = [f32, f16, bf16])] fn bench_iq2_xxs(dt: DType) -> BenchSetup { let n = 4096 * 4096usize; let n_blocks = n / 256; - BenchSetup::new(ffai_gguf_dequant_iq2_xxs::kernel_ir_for(dt)) + BenchSetup::new(mt_gguf_dequant_iq2_xxs::kernel_ir_for(dt)) .buffer(BenchBuffer::random("qs_u32", n_blocks * 16, DType::U32)) .buffer(BenchBuffer::random("d_f32", n_blocks, DType::F32)) .buffer(BenchBuffer::random("grid", 2048, DType::U8)) diff --git a/crates/metaltile-std/src/ffai/gguf_dequant_iq2_xxs_raw.rs b/crates/metaltile-std/src/kernels/quant/gguf_dequant_iq2_xxs_raw.rs similarity index 91% rename from crates/metaltile-std/src/ffai/gguf_dequant_iq2_xxs_raw.rs rename to crates/metaltile-std/src/kernels/quant/gguf_dequant_iq2_xxs_raw.rs index f820a7dd..3eb9b35f 100644 --- a/crates/metaltile-std/src/ffai/gguf_dequant_iq2_xxs_raw.rs +++ b/crates/metaltile-std/src/kernels/quant/gguf_dequant_iq2_xxs_raw.rs @@ -2,7 +2,7 @@ //! SPDX-License-Identifier: Apache-2.0 //! GGUF IQ2_XXS dequant — raw-bytes variant. //! -//! Same algorithm as `ffai_gguf_dequant_iq2_xxs` but reads `qs` from +//! Same algorithm as `mt_gguf_dequant_iq2_xxs` but reads `qs` from //! the on-disk 66-byte block layout directly, skipping the CPU //! preprocess that split each block into a separate `qs_u32` buffer. //! @@ -16,7 +16,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_gguf_dequant_iq2_xxs_raw( +pub fn mt_gguf_dequant_iq2_xxs_raw( raw_bytes: Tensor, d_f32: Tensor, grid: Tensor, @@ -70,12 +70,12 @@ pub fn ffai_gguf_dequant_iq2_xxs_raw( pub mod kernel_tests { use metaltile::test::*; - use super::ffai_gguf_dequant_iq2_xxs_raw; + use super::mt_gguf_dequant_iq2_xxs_raw; #[test] fn codegen_iq2_xxs_raw_smoke() { for dt in [DType::F32, DType::F16, DType::BF16] { - let ir = ffai_gguf_dequant_iq2_xxs_raw::kernel_ir_for(dt); + let ir = mt_gguf_dequant_iq2_xxs_raw::kernel_ir_for(dt); assert!(!ir.body.ops.is_empty(), "kernel body emitted no ops for {dt:?}"); assert!(ir.params.iter().any(|p| p.name == "raw_bytes"), "missing raw_bytes param"); } @@ -85,13 +85,13 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gguf_dequant_iq2_xxs_raw; + use super::mt_gguf_dequant_iq2_xxs_raw; #[bench(dtypes = [f32, f16, bf16])] fn bench_iq2_xxs_raw(dt: DType) -> BenchSetup { let n = 4096 * 4096usize; let n_blocks = n / 256; - BenchSetup::new(ffai_gguf_dequant_iq2_xxs_raw::kernel_ir_for(dt)) + BenchSetup::new(mt_gguf_dequant_iq2_xxs_raw::kernel_ir_for(dt)) .buffer(BenchBuffer::random("raw_bytes", n_blocks * 66, DType::U8)) .buffer(BenchBuffer::random("d_f32", n_blocks, DType::F32)) .buffer(BenchBuffer::random("grid", 2048, DType::U8)) diff --git a/crates/metaltile-std/src/ffai/gguf_dequant_q2_k.rs b/crates/metaltile-std/src/kernels/quant/gguf_dequant_q2_k.rs similarity index 94% rename from crates/metaltile-std/src/ffai/gguf_dequant_q2_k.rs rename to crates/metaltile-std/src/kernels/quant/gguf_dequant_q2_k.rs index d271669c..5dbaa6e1 100644 --- a/crates/metaltile-std/src/ffai/gguf_dequant_q2_k.rs +++ b/crates/metaltile-std/src/kernels/quant/gguf_dequant_q2_k.rs @@ -54,7 +54,7 @@ use metaltile::kernel; // Bare `#[kernel]` — see Q8_0 sibling for why; mixed concrete + // generic param dtype set doesn't fit the legacy `bench(...)` shape. #[kernel] -pub fn ffai_gguf_dequant_q2_k( +pub fn mt_gguf_dequant_q2_k( qs_packed: Tensor, scales: Tensor, d_f32: Tensor, @@ -106,13 +106,13 @@ pub fn ffai_gguf_dequant_q2_k( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_gguf_dequant_q2_k; - use crate::{quant::gguf, utils::pack_f32}; + use super::mt_gguf_dequant_q2_k; + use crate::{kernels::quant::gguf, utils::pack_f32}; fn setup(n_blocks: usize, dt: DType) -> TestSetup { let n = n_blocks * 256; let values: Vec = (0..n).map(|i| (i as f32 * 0.007 - 0.5).sin() * 1.5).collect(); - // Pack + dequant via the shared GgufFormat oracle (quant::gguf) — the + // Pack + dequant via the shared GgufFormat oracle (kernels::quant::gguf) — the // canonical q2_k_qpos map + two-level decode now live in one place the // kernel, the quantizer, and this oracle all share, so the oracle can't // drift from the kernel (the bug class fixed in #264). @@ -120,7 +120,7 @@ pub mod kernel_tests { let dequantized = gguf::dequant_q2_k(&p); // Pack u32 vec as little-endian bytes for the test framework. let qs_bytes: Vec = p.qs_packed.iter().flat_map(|w| w.to_le_bytes()).collect(); - TestSetup::new(ffai_gguf_dequant_q2_k::kernel_ir_for(dt)) + TestSetup::new(mt_gguf_dequant_q2_k::kernel_ir_for(dt)) .input(TestBuffer::from_vec("qs_packed", qs_bytes, DType::U32)) .input(TestBuffer::from_vec("scales", p.scales, DType::U8)) .input(TestBuffer::from_vec("d_f32", pack_f32(&p.d, DType::F32), DType::F32)) @@ -141,14 +141,14 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gguf_dequant_q2_k; + use super::mt_gguf_dequant_q2_k; #[bench(dtypes = [f32, f16, bf16])] fn bench_q2_k(dt: DType) -> BenchSetup { // Representative MoE-expert down-proj slab — 4096 × 4096. let n = 4096 * 4096usize; let n_blocks = n / 256; - BenchSetup::new(ffai_gguf_dequant_q2_k::kernel_ir_for(dt)) + BenchSetup::new(mt_gguf_dequant_q2_k::kernel_ir_for(dt)) .buffer(BenchBuffer::random("qs_packed", n_blocks * 16, DType::U32)) .buffer(BenchBuffer::random("scales", n_blocks * 16, DType::U8)) .buffer(BenchBuffer::random("d_f32", n_blocks, DType::F32)) diff --git a/crates/metaltile-std/src/ffai/gguf_dequant_q8_0.rs b/crates/metaltile-std/src/kernels/quant/gguf_dequant_q8_0.rs similarity index 93% rename from crates/metaltile-std/src/ffai/gguf_dequant_q8_0.rs rename to crates/metaltile-std/src/kernels/quant/gguf_dequant_q8_0.rs index d4a470f2..01fd71d9 100644 --- a/crates/metaltile-std/src/ffai/gguf_dequant_q8_0.rs +++ b/crates/metaltile-std/src/kernels/quant/gguf_dequant_q8_0.rs @@ -54,7 +54,7 @@ use metaltile::kernel; // fn below registers this kernel for `tile bench` without the legacy // path. #[kernel] -pub fn ffai_gguf_dequant_q8_0( +pub fn mt_gguf_dequant_q8_0( qs_signed: Tensor, scales: Tensor, out: Tensor, @@ -83,17 +83,17 @@ pub fn ffai_gguf_dequant_q8_0( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_gguf_dequant_q8_0; - use crate::{quant::gguf, utils::pack_f32}; + use super::mt_gguf_dequant_q8_0; + use crate::{kernels::quant::gguf, utils::pack_f32}; fn setup(n_blocks: usize, dt: DType) -> TestSetup { let n = n_blocks * 32; let values: Vec = (0..n).map(|i| (i as f32 * 0.013 - 0.4).sin() * 3.5).collect(); // Pack + dequant via the shared GgufFormat oracle — the single source of - // truth every q8_0 path decodes through (see quant::gguf). + // truth every q8_0 path decodes through (see kernels::quant::gguf). let p = gguf::pack_q8_0(&values); let dequantized = gguf::dequant_q8_0(&p); - TestSetup::new(ffai_gguf_dequant_q8_0::kernel_ir_for(dt)) + TestSetup::new(mt_gguf_dequant_q8_0::kernel_ir_for(dt)) .input(TestBuffer::from_vec("qs_signed", p.qs, DType::U8)) .input(TestBuffer::from_vec("scales", pack_f32(&p.scales, DType::F32), DType::F32)) .input(TestBuffer::zeros("out", n, dt)) @@ -112,14 +112,14 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gguf_dequant_q8_0; + use super::mt_gguf_dequant_q8_0; #[bench(dtypes = [f32, f16, bf16])] fn bench_q8_0(dt: DType) -> BenchSetup { // 4096 × 4096 attn projection slab. let n = 4096 * 4096usize; let n_blocks = n / 32; - BenchSetup::new(ffai_gguf_dequant_q8_0::kernel_ir_for(dt)) + BenchSetup::new(mt_gguf_dequant_q8_0::kernel_ir_for(dt)) .buffer(BenchBuffer::random("qs_signed", n, DType::U8)) .buffer(BenchBuffer::random("scales", n_blocks, DType::F32)) .buffer(BenchBuffer::zeros("out", n, dt).output()) diff --git a/crates/metaltile-std/src/ffai/gguf_iq2_xxs_extract_qs.rs b/crates/metaltile-std/src/kernels/quant/gguf_iq2_xxs_extract_qs.rs similarity index 90% rename from crates/metaltile-std/src/ffai/gguf_iq2_xxs_extract_qs.rs rename to crates/metaltile-std/src/kernels/quant/gguf_iq2_xxs_extract_qs.rs index 91b4b1c6..115c9906 100644 --- a/crates/metaltile-std/src/ffai/gguf_iq2_xxs_extract_qs.rs +++ b/crates/metaltile-std/src/kernels/quant/gguf_iq2_xxs_extract_qs.rs @@ -7,7 +7,7 @@ //! 3 weight roles on DSv4-Flash). The extract kernel reads 4 raw //! bytes from the on-disk 66-byte block layout, assembles one u32, //! and writes to the packed `qs_u32` staging buffer the existing -//! `ffai_gguf_dequant_iq2_xxs` kernel expects. +//! `mt_gguf_dequant_iq2_xxs` kernel expects. //! //! ## Block layout //! @@ -34,7 +34,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_gguf_iq2_xxs_extract_qs( +pub fn mt_gguf_iq2_xxs_extract_qs( raw_bytes: Tensor, mut qs_u32: Tensor, #[constexpr] n_blocks: u32, @@ -57,11 +57,11 @@ pub fn ffai_gguf_iq2_xxs_extract_qs( #[cfg(test)] pub mod kernel_tests { - use super::ffai_gguf_iq2_xxs_extract_qs; + use super::mt_gguf_iq2_xxs_extract_qs; #[test] fn codegen_extract_qs_smoke() { - let ir = ffai_gguf_iq2_xxs_extract_qs::kernel_ir_for(); + let ir = mt_gguf_iq2_xxs_extract_qs::kernel_ir_for(); assert!(!ir.body.ops.is_empty(), "kernel body emitted no ops"); assert!(ir.params.iter().any(|p| p.name == "raw_bytes"), "missing raw_bytes"); assert!(ir.params.iter().any(|p| p.name == "qs_u32"), "missing qs_u32"); @@ -71,13 +71,13 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_gguf_iq2_xxs_extract_qs; + use super::mt_gguf_iq2_xxs_extract_qs; #[bench(dtypes = [u32])] fn bench_extract_qs(_dt: DType) -> BenchSetup { let n = 4096 * 4096usize; let n_blocks = n / 256; - BenchSetup::new(ffai_gguf_iq2_xxs_extract_qs::kernel_ir_for()) + BenchSetup::new(mt_gguf_iq2_xxs_extract_qs::kernel_ir_for()) .buffer(BenchBuffer::random("raw_bytes", n_blocks * 66, DType::U8)) .buffer(BenchBuffer::zeros("qs_u32", n_blocks * 16, DType::U32).output()) .constexpr("n_blocks", n_blocks as u32) diff --git a/crates/metaltile-std/src/quant/mod.rs b/crates/metaltile-std/src/kernels/quant/mod.rs similarity index 75% rename from crates/metaltile-std/src/quant/mod.rs rename to crates/metaltile-std/src/kernels/quant/mod.rs index ea4bec30..835fe02b 100644 --- a/crates/metaltile-std/src/quant/mod.rs +++ b/crates/metaltile-std/src/kernels/quant/mod.rs @@ -23,3 +23,19 @@ pub mod codec; pub mod format; pub mod gguf; + +// Dequant kernels migrated from ffai/ + mlx/ (the codec/format above are the +// host-side source of truth these GPU kernels mirror). +pub mod aura_dequant_rotated; +pub mod aura_encode; +pub mod block_scaled_dequant; +pub mod dequant_gather; +pub mod dequant_gather_block_scaled; +pub mod dequant_q4; +pub mod fp8_block_dequant; +pub mod gguf_dequant_iq2_xxs; +pub mod gguf_dequant_iq2_xxs_raw; +pub mod gguf_dequant_q2_k; +pub mod gguf_dequant_q8_0; +pub mod gguf_iq2_xxs_extract_qs; +pub mod mxfp4_dequant; diff --git a/crates/metaltile-std/src/ffai/dsv4_mxfp4_dequant.rs b/crates/metaltile-std/src/kernels/quant/mxfp4_dequant.rs similarity index 95% rename from crates/metaltile-std/src/ffai/dsv4_mxfp4_dequant.rs rename to crates/metaltile-std/src/kernels/quant/mxfp4_dequant.rs index 7697195c..c23cfbe0 100644 --- a/crates/metaltile-std/src/ffai/dsv4_mxfp4_dequant.rs +++ b/crates/metaltile-std/src/kernels/quant/mxfp4_dequant.rs @@ -71,7 +71,7 @@ use metaltile::kernel; // Bare `#[kernel]` — mixed-dtype param set (concrete u32/f32 + generic T). #[kernel] -pub fn ffai_dsv4_mxfp4_dequant( +pub fn mt_mxfp4_dequant( qs_packed: Tensor, scales: Tensor, lut: Tensor, @@ -98,13 +98,13 @@ pub fn ffai_dsv4_mxfp4_dequant( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_mxfp4_dequant; - use crate::{quant::codec, utils::pack_f32}; + use super::mt_mxfp4_dequant; + use crate::{kernels::quant::codec, utils::pack_f32}; /// The canonical OCP-spec MXFP4 value table — `code → magnitude` for the 16 /// E2M1 codes. DSv4 ships these MoE expert weights as native MXFP4 in the /// safetensors checkpoint, but the element decode is the *same* OCP E2M1 - /// codebook as `quant::format`'s `Mxfp4` and the GGUF MXFP4 path: all route + /// codebook as `kernels::quant::format`'s `Mxfp4` and the GGUF MXFP4 path: all route /// through `codec::e2m1_decode` — the single source of truth — so the /// uploaded LUT, the host quantizer, and this oracle can't drift apart. fn mxfp4_lut() -> [f32; 16] { std::array::from_fn(|c| codec::e2m1_decode(c as u8)) } @@ -182,7 +182,7 @@ pub mod kernel_tests { let dequantized = cpu_dequant(&qs_packed, &scales); // Pack u32 vec as little-endian bytes for the test framework. let qs_bytes: Vec = qs_packed.iter().flat_map(|w| w.to_le_bytes()).collect(); - TestSetup::new(ffai_dsv4_mxfp4_dequant::kernel_ir_for(dt)) + TestSetup::new(mt_mxfp4_dequant::kernel_ir_for(dt)) .input(TestBuffer::from_vec("qs_packed", qs_bytes, DType::U32)) .input(TestBuffer::from_vec("scales", pack_f32(&scales, DType::F32), DType::F32)) .input(TestBuffer::from_vec("lut", pack_f32(&mxfp4_lut(), DType::F32), DType::F32)) @@ -202,7 +202,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_mxfp4_dequant; + use super::mt_mxfp4_dequant; #[bench(dtypes = [f32, f16, bf16])] fn bench_mxfp4(dt: DType) -> BenchSetup { @@ -210,7 +210,7 @@ pub mod kernel_benches { // expert intermediate=2048, hidden=4096). let n = 4096 * 2048usize; let n_blocks = n / 32; - BenchSetup::new(ffai_dsv4_mxfp4_dequant::kernel_ir_for(dt)) + BenchSetup::new(mt_mxfp4_dequant::kernel_ir_for(dt)) .buffer(BenchBuffer::random("qs_packed", n_blocks * 4, DType::U32)) .buffer(BenchBuffer::random("scales", n_blocks, DType::F32)) .buffer(BenchBuffer::random("lut", 16, DType::F32)) diff --git a/crates/metaltile-std/src/ffai/attn_head_gate.rs b/crates/metaltile-std/src/kernels/sdpa/attn_head_gate.rs similarity index 95% rename from crates/metaltile-std/src/ffai/attn_head_gate.rs rename to crates/metaltile-std/src/kernels/sdpa/attn_head_gate.rs index 39e4bb4c..e06d64c9 100644 --- a/crates/metaltile-std/src/ffai/attn_head_gate.rs +++ b/crates/metaltile-std/src/kernels/sdpa/attn_head_gate.rs @@ -40,7 +40,7 @@ use metaltile::kernel; // fit this 3-input + 1-constexpr shape; the new declarative `#[bench]` // on `kernel_benches::bench_attn_head_gate` below handles registration. #[kernel] -pub fn ffai_attn_head_gate( +pub fn mt_attn_head_gate( attn: Tensor, gate: Tensor, out: Tensor, @@ -63,7 +63,7 @@ pub fn ffai_attn_head_gate( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_attn_head_gate; + use super::mt_attn_head_gate; use crate::utils::{pack_f32, unpack_f32}; fn setup(n_heads: usize, head_dim: usize, dt: DType) -> TestSetup { @@ -79,7 +79,7 @@ pub mod kernel_tests { expected.push(a_dt[h * head_dim + d] * s); } } - TestSetup::new(ffai_attn_head_gate::kernel_ir_for(dt)) + TestSetup::new(mt_attn_head_gate::kernel_ir_for(dt)) .input(TestBuffer::from_vec("attn", pack_f32(&attn, dt), dt)) .input(TestBuffer::from_vec("gate", pack_f32(&gate, dt), dt)) .input(TestBuffer::zeros("out", n, dt)) @@ -101,14 +101,14 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_attn_head_gate; + use super::mt_attn_head_gate; #[bench(dtypes = [f32, f16, bf16])] fn bench_attn_head_gate(dt: DType) -> BenchSetup { // Step-3 full-attn shape. let (n_heads, head_dim) = (64usize, 128usize); let n = n_heads * head_dim; - BenchSetup::new(ffai_attn_head_gate::kernel_ir_for(dt)) + BenchSetup::new(mt_attn_head_gate::kernel_ir_for(dt)) .buffer(BenchBuffer::random("attn", n, dt)) .buffer(BenchBuffer::random("gate", n_heads, dt)) .buffer(BenchBuffer::zeros("out", n, dt).output()) diff --git a/crates/metaltile-std/src/ffai/aura_flash_p1.rs b/crates/metaltile-std/src/kernels/sdpa/aura_flash_p1.rs similarity index 96% rename from crates/metaltile-std/src/ffai/aura_flash_p1.rs rename to crates/metaltile-std/src/kernels/sdpa/aura_flash_p1.rs index 19fad41d..0af5e0bb 100644 --- a/crates/metaltile-std/src/ffai/aura_flash_p1.rs +++ b/crates/metaltile-std/src/kernels/sdpa/aura_flash_p1.rs @@ -29,7 +29,7 @@ //! - `m_partials [q_heads, num_blocks]` f32 //! - `l_partials [q_heads, num_blocks]` f32 //! -//! `aura_flash_pass2` later reduces the partials cross-block. +//! `mt_aura_flash_pass2` later reduces the partials cross-block. //! //! ## Dispatch //! @@ -54,10 +54,10 @@ use metaltile::kernel; /// AURA flash pass-1 (non-causal): per-block online-softmax, KB=4. /// -/// Produces kernels: `aura_flash_p1_kb4_vb2_d64`, `_kb4_vb2_d128`, +/// Produces kernels: `mt_aura_flash_p1_kb4_vb2_d64`, `_kb4_vb2_d128`, /// `_kb4_vb4_d64`, `_kb4_vb4_d128`. #[kernel(variants(VB = [2, 2, 4, 4], DIM = [64, 128, 64, 128], VL = [4, 4, 16, 16], DPL = [2, 4, 2, 4], suffix = "kb4_vb{VB}_d{DIM}"))] -pub fn aura_flash_p1( +pub fn mt_aura_flash_p1( q_rot: Tensor, key_packed: Tensor, key_norms: Tensor, @@ -225,14 +225,14 @@ pub fn aura_flash_p1( /// AURA flash pass-1 (causal): per-block online-softmax, KB=4, VB=2. /// -/// Same compressed-domain online-softmax as `aura_flash_p1`, with +/// Same compressed-domain online-softmax as `mt_aura_flash_p1`, with /// the per-token loop clamped at `q_position + 1` — every key strictly /// after the query token is masked out. Production recipe aura4v2. /// -/// Produces kernels: `aura_flash_p1_causal_kb4_vb2_d64`, +/// Produces kernels: `mt_aura_flash_p1_causal_kb4_vb2_d64`, /// `_causal_kb4_vb2_d128`. #[kernel(variants(DIM = [64, 128], DPL = [2, 4], suffix = "kb4_vb2_d{DIM}"))] -pub fn aura_flash_p1_causal( +pub fn mt_aura_flash_p1_causal( q_rot: Tensor, key_packed: Tensor, key_norms: Tensor, @@ -373,17 +373,17 @@ pub fn aura_flash_p1_causal( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::{aura_flash_p1_causal_kb4_vb2_d128, aura_flash_p1_kb4_vb2_d128}; + use super::{mt_aura_flash_p1_causal_kb4_vb2_d128, mt_aura_flash_p1_kb4_vb2_d128}; use crate::utils::{pack_f32, unpack_f32}; // ── GPU pass-1 partials, validated directly ───────────────────────── // - // The companion `aura_flash_pass2` test CPU-emulates pass 1 to stage + // The companion `mt_aura_flash_pass2` test CPU-emulates pass 1 to stage // the partials it feeds the GPU reducer — so the GPU pass-1 path itself - // was never exercised. This test dispatches the real `aura_flash_p1` + // was never exercised. This test dispatches the real `mt_aura_flash_p1` // kernel and validates its three partial outputs against a CPU oracle: // the same per-(q_head, block) online-softmax block reduction over the - // AURA-codebook-decoded K,V that `aura_flash_pass2`'s `emulate_p1` + // AURA-codebook-decoded K,V that `mt_aura_flash_pass2`'s `emulate_p1` // reference computes. All three partials are deterministic per // (q_head, block), so all three are checked. @@ -427,12 +427,12 @@ pub mod kernel_tests { packed } - /// CPU oracle for `aura_flash_p1` (non-causal): per (q_head, block) + /// CPU oracle for `mt_aura_flash_p1` (non-causal): per (q_head, block) /// online-softmax over the block's token range, K/V decoded from /// `codebook[index] * norm`. Emits the partials exactly as the kernel /// stores them — `o` un-normalised accumulator `[q_head, block, dim]`, /// `m` block max `[q_head, block]`, `l` block sum_exp `[q_head, block]`. - /// Mirrors `aura_flash_pass2::kernel_tests::emulate_p1`. + /// Mirrors `mt_aura_flash_pass2::kernel_tests::emulate_p1`. #[allow(clippy::too_many_arguments)] fn emulate_p1( q_rot: &[f32], @@ -532,7 +532,7 @@ pub mod kernel_tests { let m_part = unpack_f32(&pack_f32(&m_part, dt), dt); let l_part = unpack_f32(&pack_f32(&l_part, dt), dt); - TestSetup::new(aura_flash_p1_kb4_vb2_d128::kernel_ir_for(dt)) + TestSetup::new(mt_aura_flash_p1_kb4_vb2_d128::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("q_rot", pack_f32(&q_rot, dt), dt)) .input(TestBuffer::from_vec("key_packed", pack_u32(&key_packed), DType::U32)) @@ -565,7 +565,7 @@ pub mod kernel_tests { // ── Causal pass-1 partials ────────────────────────────────────────── // - // The causal sibling `aura_flash_p1_causal_kb4_vb2_d128` shares the whole + // The causal sibling `mt_aura_flash_p1_causal_kb4_vb2_d128` shares the whole // online-softmax body but clamps its per-token loop at `q_position + 1` — // keys strictly after the query position contribute nothing. With a // mid-stream `q_position`, the block containing the cutoff sums only its @@ -623,7 +623,7 @@ pub mod kernel_tests { let m_part = unpack_f32(&pack_f32(&m_part, dt), dt); let l_part = unpack_f32(&pack_f32(&l_part, dt), dt); - TestSetup::new(aura_flash_p1_causal_kb4_vb2_d128::kernel_ir_for(dt)) + TestSetup::new(mt_aura_flash_p1_causal_kb4_vb2_d128::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("q_rot", pack_f32(&q_rot, dt), dt)) .input(TestBuffer::from_vec("key_packed", pack_u32(&key_packed), DType::U32)) @@ -715,7 +715,7 @@ pub mod kernel_benches { variants(VB = [2, 2, 4, 4], DIM = [64, 128, 64, 128], suffix = "kb4_vb{VB}_d{DIM}"))] fn bench_flash_p1(dt: DType) -> BenchSetup { flash_p1( - BenchSetup::new(aura_flash_p1_kb4_vbVB_dDIM::kernel_ir_for(dt)), + BenchSetup::new(mt_aura_flash_p1_kb4_vbVB_dDIM::kernel_ir_for(dt)), dt, DIM, 4, @@ -729,7 +729,7 @@ pub mod kernel_benches { variants(DIM = [64, 128], suffix = "kb4_vb2_d{DIM}"))] fn bench_flash_p1_causal(dt: DType) -> BenchSetup { flash_p1( - BenchSetup::new(aura_flash_p1_causal_kb4_vb2_dDIM::kernel_ir_for(dt)), + BenchSetup::new(mt_aura_flash_p1_causal_kb4_vb2_dDIM::kernel_ir_for(dt)), dt, DIM, 4, diff --git a/crates/metaltile-std/src/ffai/aura_flash_pass2.rs b/crates/metaltile-std/src/kernels/sdpa/aura_flash_pass2.rs similarity index 96% rename from crates/metaltile-std/src/ffai/aura_flash_pass2.rs rename to crates/metaltile-std/src/kernels/sdpa/aura_flash_pass2.rs index 6a54841a..9eb60b05 100644 --- a/crates/metaltile-std/src/ffai/aura_flash_pass2.rs +++ b/crates/metaltile-std/src/kernels/sdpa/aura_flash_pass2.rs @@ -3,7 +3,7 @@ //! AURA Flash Pass 2 — cross-block online-softmax merge. //! //! Reduces the `(o_partials, m_partials, l_partials)` tuples emitted -//! by `aura_flash_p1` (one tuple per (q_idx, block_idx) pair) into a +//! by `mt_aura_flash_p1` (one tuple per (q_idx, block_idx) pair) into a //! single `(o, m, l)` per q_idx, then writes the final attention //! output `o / l` cast to bf16. //! @@ -46,13 +46,13 @@ use metaltile::kernel; /// AURA flash pass-2 cross-block softmax merge. /// -/// Produces kernels: `aura_flash_pass2_d64`, `_d80`, `_d96`, `_d128`, +/// Produces kernels: `mt_aura_flash_pass2_d64`, `_d80`, `_d96`, `_d128`, /// `_d256`, `_d512`. /// /// Grid: `(q_heads, 1, 1)` with a 32-lane reduction threadgroup; only /// `dim` (and the buffer sizes it drives) varies. #[kernel(variants(DIM = [64, 80, 96, 128, 256, 512], DPL = [2, 3, 3, 4, 8, 16], suffix = "d{DIM}"))] -pub fn aura_flash_pass2( +pub fn mt_aura_flash_pass2( o_partials: Tensor, m_partials: Tensor, l_partials: Tensor, @@ -118,14 +118,14 @@ pub fn aura_flash_pass2( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::aura_flash_pass2_d128; + use super::mt_aura_flash_pass2_d128; use crate::utils::{pack_f32, unpack_f32}; // ── COMBINED AURA flash decode, exercised through pass 2 ───────────── // // The test runner dispatches a single kernel per `TestSetup`, so we // cannot chain p1 → pass2 on the GPU. Instead we emulate the - // non-causal `aura_flash_p1` per-block online-softmax on the CPU + // non-causal `mt_aura_flash_p1` per-block online-softmax on the CPU // (decoding K/V from `codebook[index] * norm`), producing the // `(o_partials, m_partials, l_partials)` staging tuples, then // dispatch the real GPU pass-2 reducer. Its output must equal a dense @@ -137,7 +137,7 @@ pub mod kernel_tests { const KEY_BITS: usize = 4; const VALUE_BITS: usize = 4; - /// Emulate `aura_flash_p1` (non-causal): per (q_head, block) + /// Emulate `mt_aura_flash_p1` (non-causal): per (q_head, block) /// online-softmax over the block's token range, K/V decoded from /// `codebook[index] * norm`. Emits per-block partials exactly as p1 /// stores them (o is the un-normalised accumulator, m the block max, @@ -278,7 +278,7 @@ pub mod kernel_tests { kv_heads, tokens, dim, ); - TestSetup::new(aura_flash_pass2_d128::kernel_ir_for(dt)) + TestSetup::new(mt_aura_flash_pass2_d128::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("o_partials", pack_f32(&o_part, dt), dt)) .input(TestBuffer::from_vec("m_partials", pack_f32(&m_part, dt), dt)) @@ -316,6 +316,6 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16], variants(DIM = [64, 80, 96, 128, 256, 512], suffix = "d{DIM}"))] fn bench_flash_pass2(dt: DType) -> BenchSetup { - flash_pass2(BenchSetup::new(aura_flash_pass2_dDIM::kernel_ir_for(dt)), DIM, dt) + flash_pass2(BenchSetup::new(mt_aura_flash_pass2_dDIM::kernel_ir_for(dt)), DIM, dt) } } diff --git a/crates/metaltile-std/src/ffai/aura_flash_sdpa.rs b/crates/metaltile-std/src/kernels/sdpa/aura_flash_sdpa.rs similarity index 96% rename from crates/metaltile-std/src/ffai/aura_flash_sdpa.rs rename to crates/metaltile-std/src/kernels/sdpa/aura_flash_sdpa.rs index 61356362..445fef28 100644 --- a/crates/metaltile-std/src/ffai/aura_flash_sdpa.rs +++ b/crates/metaltile-std/src/kernels/sdpa/aura_flash_sdpa.rs @@ -5,14 +5,14 @@ //! and sliding-window causal masking. Port of `turbo_flash_sdpa.h` //! (spec 041 phase 1.1, GPT-OSS sink-attention family). //! -//! Unlike the `aura_flash_p1` + `aura_flash_pass2` pair, this does the +//! Unlike the `mt_aura_flash_p1` + `mt_aura_flash_pass2` pair, this does the //! whole attention in one dispatch — one threadgroup (a single //! 32-lane simdgroup) per query, iterating every K/V token with a //! running online softmax, then writing the normalized output. This //! side-steps the pass2-with-sinks graph-fusion incoherence that the //! two-pass β-with-sinks drafts hit on GPT-OSS-20B. //! -//! Layout (matches `aura_flash_p1` / `aura_score` / `aura_value` — all +//! Layout (matches `mt_aura_flash_p1` / `mt_aura_score` / `mt_aura_value` — all //! generic over `T` for the auxiliary float buffers; internal math stays //! f32 via cast-at-load. See header note on the dtype unification): //! - q_rot: [B*nQ, dim] T (WHT-rotated + pre-scaled by caller) @@ -27,8 +27,8 @@ //! //! Norms / codebook were f32-typed in the original port (spec 041 //! phase 1.1), inherited from `mlx-swift-lm`'s TurboQuant codec. The -//! sibling `aura_flash_p1` / `aura_flash_pass2` / `aura_score` / -//! `aura_value` kernels were genericised to `Tensor` during the bf16 +//! sibling `mt_aura_flash_p1` / `mt_aura_flash_pass2` / `mt_aura_score` / +//! `mt_aura_value` kernels were genericised to `Tensor` during the bf16 //! coverage rollout; this kernel was the last laggard. Unifying the //! dtype contract lets FFAI's cache store norms+codebook in the //! activation dtype directly — no per-call cast on the decode hot path. @@ -44,7 +44,7 @@ //! Lane `program_id::<0>()` ∈ [0,32) owns dim slots `lane + i*32`; //! `program_id::<1>()` = query index. The MLX reference fans tokens //! across 32 simdgroups; this port keeps the simpler single-simdgroup -//! shape of `aura_flash_p1` (correctness-equivalent; token-parallelism +//! shape of `mt_aura_flash_p1` (correctness-equivalent; token-parallelism //! is a perf follow-up). //! //! ## DISPATCH INVARIANTS @@ -65,10 +65,10 @@ use metaltile::kernel; /// AURA fused single-pass SDPA, KB=4. /// -/// Produces kernels: `aura_flash_sdpa_kb4_vb2_d64`, `_kb4_vb2_d128`, +/// Produces kernels: `mt_aura_flash_sdpa_kb4_vb2_d64`, `_kb4_vb2_d128`, /// `_kb4_vb4_d64`, `_kb4_vb4_d128`. #[kernel(variants(VB = [2, 2, 4, 4], DIM = [64, 128, 64, 128], VL = [4, 4, 16, 16], DPL = [2, 4, 2, 4], suffix = "kb4_vb{VB}_d{DIM}"))] -pub fn aura_flash_sdpa( +pub fn mt_aura_flash_sdpa( q_rot: Tensor, key_packed: Tensor, key_norms: Tensor, @@ -215,7 +215,7 @@ pub fn aura_flash_sdpa( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::aura_flash_sdpa_kb4_vb4_d128; + use super::mt_aura_flash_sdpa_kb4_vb4_d128; use crate::utils::{pack_f32, unpack_f32}; fn u32_bytes(v: &[u32]) -> Vec { v.iter().flat_map(|x| x.to_le_bytes()).collect() } @@ -385,7 +385,7 @@ pub mod kernel_tests { window_size, ); - TestSetup::new(aura_flash_sdpa_kb4_vb4_d128::kernel_ir_for(dt)) + TestSetup::new(mt_aura_flash_sdpa_kb4_vb4_d128::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("q_rot", pack_f32(&q_rot, dt), dt)) .input(TestBuffer::from_vec("key_packed", u32_bytes(&key_packed), DType::U32)) @@ -496,6 +496,6 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16], variants(VB = [2, 2, 4, 4], DIM = [64, 128, 64, 128], suffix = "kb4_vb{VB}_d{DIM}"))] fn bench_sdpa(dt: DType) -> BenchSetup { - setup(aura_flash_sdpa_kb4_vbVB_dDIM::kernel_ir_for(dt), DIM, 4, VB, dt) + setup(mt_aura_flash_sdpa_kb4_vbVB_dDIM::kernel_ir_for(dt), DIM, 4, VB, dt) } } diff --git a/crates/metaltile-std/src/ffai/aura_score.rs b/crates/metaltile-std/src/kernels/sdpa/aura_score.rs similarity index 96% rename from crates/metaltile-std/src/ffai/aura_score.rs rename to crates/metaltile-std/src/kernels/sdpa/aura_score.rs index 8e65c68b..f2df08e8 100644 --- a/crates/metaltile-std/src/ffai/aura_score.rs +++ b/crates/metaltile-std/src/kernels/sdpa/aura_score.rs @@ -49,8 +49,8 @@ use metaltile::kernel; /// AURA quantized attention-score kernel — variable bit-widths (2, 3, 4, 6, 8). /// -/// Produces kernels: `aura_score_int2`, `aura_score_int3`, `aura_score_int4`, -/// `aura_score_int6`, `aura_score_int8`. +/// Produces kernels: `mt_aura_score_int2`, `mt_aura_score_int3`, `mt_aura_score_int4`, +/// `mt_aura_score_int6`, `mt_aura_score_int8`. /// /// Lane-strided dot-product of the query vector against the BITS-wide packed /// key vector: decodes codes via the straddle-aware two-word bit-stream formula, @@ -59,7 +59,7 @@ use metaltile::kernel; /// /// Grid: Grid3D, `[n_q * n_kv_heads / repeat, n_tokens, 1]`, tpg = 32. #[kernel(variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] -pub fn aura_score( +pub fn mt_aura_score( q_rot: Tensor, packed: Tensor, norms: Tensor, @@ -124,7 +124,7 @@ pub fn aura_score( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::aura_score_int4; + use super::mt_aura_score_int4; use crate::utils::{pack_f32, unpack_f32}; fn round(v: f32, dt: DType) -> f32 { unpack_f32(&pack_f32(&[v], dt), dt)[0] } @@ -180,7 +180,7 @@ pub mod kernel_tests { } } - TestSetup::new(aura_score_int4::kernel_ir_for(dt)) + TestSetup::new(mt_aura_score_int4::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q_rot", pack_f32(&q_rot_r, dt), dt)) .input(TestBuffer::from_vec("packed", pack_u32(&packed), DType::U32)) @@ -232,6 +232,6 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16], variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] fn bench_aura_score(dt: DType) -> BenchSetup { - setup(BenchSetup::new(aura_score_intBITS::kernel_ir_for(dt)), 128, BITS, 32, 8, 4096, dt) + setup(BenchSetup::new(mt_aura_score_intBITS::kernel_ir_for(dt)), 128, BITS, 32, 8, 4096, dt) } } diff --git a/crates/metaltile-std/src/ffai/aura_value.rs b/crates/metaltile-std/src/kernels/sdpa/aura_value.rs similarity index 96% rename from crates/metaltile-std/src/ffai/aura_value.rs rename to crates/metaltile-std/src/kernels/sdpa/aura_value.rs index 58453c24..29179724 100644 --- a/crates/metaltile-std/src/ffai/aura_value.rs +++ b/crates/metaltile-std/src/kernels/sdpa/aura_value.rs @@ -34,8 +34,8 @@ use metaltile::kernel; #[rustfmt::skip] /// AURA quantized value-aggregation kernel — variable bit-widths (2, 3, 4, 6, 8). /// -/// Produces kernels: `aura_value_int2`, `aura_value_int3`, `aura_value_int4`, -/// `aura_value_int6`, `aura_value_int8`. +/// Produces kernels: `mt_aura_value_int2`, `mt_aura_value_int3`, `mt_aura_value_int4`, +/// `mt_aura_value_int6`, `mt_aura_value_int8`. /// /// For each (dim, head) thread: unpacks the `BITS`-wide code at position `d` /// from the LSB-first bit-stream, fetches the codebook centroid, and accumulates @@ -43,7 +43,7 @@ use metaltile::kernel; /// /// Grid: Grid3D, `[dim, n_heads, 1]`. #[kernel(variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] -pub fn aura_value( +pub fn mt_aura_value( weights: Tensor, packed: Tensor, norms: Tensor, @@ -95,7 +95,7 @@ pub fn aura_value( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::aura_value_int4; + use super::mt_aura_value_int4; use crate::utils::{pack_f32, unpack_f32}; fn round(v: f32, dt: DType) -> f32 { unpack_f32(&pack_f32(&[v], dt), dt)[0] } @@ -157,7 +157,7 @@ pub mod kernel_tests { } } - TestSetup::new(aura_value_int4::kernel_ir_for(dt)) + TestSetup::new(mt_aura_value_int4::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("weights", pack_f32(&weights_r, dt), dt)) .input(TestBuffer::from_vec("packed", pack_u32(&packed), DType::U32)) @@ -230,6 +230,6 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16], variants(BITS = [2, 3, 4, 6, 8], suffix = "int{BITS}"))] fn bench_aura_value(dt: DType) -> BenchSetup { - setup(BenchSetup::new(aura_value_intBITS::kernel_ir_for(dt)), 128, BITS, 32, 8, 4096, dt) + setup(BenchSetup::new(mt_aura_value_intBITS::kernel_ir_for(dt)), 128, BITS, 32, 8, 4096, dt) } } diff --git a/crates/metaltile-std/src/ffai/dsv4_compressor_pool.rs b/crates/metaltile-std/src/kernels/sdpa/compressor_pool.rs similarity index 95% rename from crates/metaltile-std/src/ffai/dsv4_compressor_pool.rs rename to crates/metaltile-std/src/kernels/sdpa/compressor_pool.rs index 35f6f978..e8cb3ce4 100644 --- a/crates/metaltile-std/src/ffai/dsv4_compressor_pool.rs +++ b/crates/metaltile-std/src/kernels/sdpa/compressor_pool.rs @@ -48,7 +48,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_dsv4_compressor_pool( +pub fn mt_compressor_pool( raw_kv: Tensor, gate: Tensor, ape: Tensor, @@ -87,7 +87,7 @@ pub fn ffai_dsv4_compressor_pool( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_compressor_pool; + use super::mt_compressor_pool; use crate::utils::{pack_f32, unpack_f32}; fn cpu_reference( @@ -127,7 +127,7 @@ pub mod kernel_tests { let raw_dt = unpack_f32(&pack_f32(&raw_kv, dt), dt); let ape_dt = unpack_f32(&pack_f32(&ape, dt), dt); let expected = cpu_reference(&raw_dt, &gate, &ape_dt, pool_len, head_dim); - TestSetup::new(ffai_dsv4_compressor_pool::kernel_ir_for(dt)) + TestSetup::new(mt_compressor_pool::kernel_ir_for(dt)) .input(TestBuffer::from_vec("raw_kv", pack_f32(&raw_kv, dt), dt)) .input(TestBuffer::from_vec("gate", pack_f32(&gate, DType::F32), DType::F32)) .input(TestBuffer::from_vec("ape", pack_f32(&ape, dt), dt)) @@ -150,12 +150,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_compressor_pool; + use super::mt_compressor_pool; #[bench(dtypes = [f32, f16, bf16])] fn bench_csa(dt: DType) -> BenchSetup { let (pool, head_dim) = (8usize, 512usize); - BenchSetup::new(ffai_dsv4_compressor_pool::kernel_ir_for(dt)) + BenchSetup::new(mt_compressor_pool::kernel_ir_for(dt)) .buffer(BenchBuffer::random("raw_kv", pool * head_dim, dt)) .buffer(BenchBuffer::random("gate", pool, DType::F32)) .buffer(BenchBuffer::random("ape", pool * head_dim, dt)) @@ -172,7 +172,7 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16])] fn bench_hca(dt: DType) -> BenchSetup { let (pool, head_dim) = (128usize, 512usize); - BenchSetup::new(ffai_dsv4_compressor_pool::kernel_ir_for(dt)) + BenchSetup::new(mt_compressor_pool::kernel_ir_for(dt)) .buffer(BenchBuffer::random("raw_kv", pool * head_dim, dt)) .buffer(BenchBuffer::random("gate", pool, DType::F32)) .buffer(BenchBuffer::random("ape", pool * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/dsv4_csa_sdpa_decode.rs b/crates/metaltile-std/src/kernels/sdpa/csa_sdpa_decode.rs similarity index 96% rename from crates/metaltile-std/src/ffai/dsv4_csa_sdpa_decode.rs rename to crates/metaltile-std/src/kernels/sdpa/csa_sdpa_decode.rs index 1fe003e7..4a0d39d2 100644 --- a/crates/metaltile-std/src/ffai/dsv4_csa_sdpa_decode.rs +++ b/crates/metaltile-std/src/kernels/sdpa/csa_sdpa_decode.rs @@ -3,13 +3,13 @@ //! DSv4 CSA sparse-gather SDPA decode for `head_dim == 512`. //! //! Single-token attention over a **selected subset** of KV cache -//! positions. Clone of [`crate::ffai::sdpa_decode_d512`] with the +//! positions. Clone of [`crate::kernels::sdpa::sdpa_decode`] with the //! dense `for _t in 0..n_kv` inner loop replaced by an index-gather //! over a caller-supplied `selected_indices[n_selected]` buffer. //! //! The caller builds `selected_indices` by unioning: //! - the top-512 indices returned by the Lightning Indexer -//! (`ffai_dsv4_indexer_topk_block`) +//! (`mt_indexer_topk_block`) //! - the trailing `sliding_window` (DSv4 default: 128) most-recent //! cache positions //! @@ -39,7 +39,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_dsv4_csa_sdpa_decode( +pub fn mt_csa_sdpa_decode( q: Tensor, k: Tensor, v: Tensor, @@ -301,12 +301,12 @@ mod tests { core::{DType, ir::KernelMode}, }; - use super::ffai_dsv4_csa_sdpa_decode; + use super::mt_csa_sdpa_decode; fn msl_for(dt: DType) -> String { - let mut k = ffai_dsv4_csa_sdpa_decode::kernel_ir_for(dt); + let mut k = mt_csa_sdpa_decode::kernel_ir_for(dt); k.mode = KernelMode::Reduction; - MslGenerator::default().generate(&k).expect("ffai_dsv4_csa_sdpa_decode codegen succeeds") + MslGenerator::default().generate(&k).expect("mt_csa_sdpa_decode codegen succeeds") } #[test] @@ -315,8 +315,8 @@ mod tests { let src = msl_for(dt); assert!(!src.trim().is_empty(), "MSL for {dt:?} should not be empty"); assert!( - src.contains("kernel void ffai_dsv4_csa_sdpa_decode"), - "MSL for {dt:?} should declare ffai_dsv4_csa_sdpa_decode:\n{src}", + src.contains("kernel void mt_csa_sdpa_decode"), + "MSL for {dt:?} should declare mt_csa_sdpa_decode:\n{src}", ); } } @@ -325,7 +325,7 @@ mod tests { pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_csa_sdpa_decode; + use super::mt_csa_sdpa_decode; use crate::utils::{pack_f32, unpack_f32}; /// Sparse-gather oracle: dense SDPA but only over the listed cache @@ -402,7 +402,7 @@ pub mod kernel_tests { &q, &k, &v, &selected, n_q_heads, n_kv_heads, head_dim, kv_stride, scale, ); let sel_bytes: Vec = selected.iter().flat_map(|i| i.to_le_bytes()).collect(); - TestSetup::new(ffai_dsv4_csa_sdpa_decode::kernel_ir_for(dt)) + TestSetup::new(mt_csa_sdpa_decode::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -422,7 +422,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_csa_sdpa_decode; + use super::mt_csa_sdpa_decode; #[bench(dtypes = [f32, f16, bf16])] fn bench_csa(dt: DType) -> BenchSetup { @@ -433,7 +433,7 @@ pub mod kernel_benches { let bytes = (2 * n_q_heads * head_dim + 2 * n_kv_heads * kv_stride * head_dim) * dt.size_bytes() + n_selected * 4; - BenchSetup::new(ffai_dsv4_csa_sdpa_decode::kernel_ir_for(dt)) + BenchSetup::new(mt_csa_sdpa_decode::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/flash_block_scaled_sdpa.rs b/crates/metaltile-std/src/kernels/sdpa/flash_block_scaled_sdpa.rs similarity index 99% rename from crates/metaltile-std/src/ffai/flash_block_scaled_sdpa.rs rename to crates/metaltile-std/src/kernels/sdpa/flash_block_scaled_sdpa.rs index f919b2f6..fc3a13f3 100644 --- a/crates/metaltile-std/src/ffai/flash_block_scaled_sdpa.rs +++ b/crates/metaltile-std/src/kernels/sdpa/flash_block_scaled_sdpa.rs @@ -3,7 +3,7 @@ //! Flash **block-scaled** SDPA — single-pass online-softmax attention over a //! block-scaled-quantized K/V cache, for the spec-conformant formats //! (mxfp4 / nvfp4 / mxfp8 e4m3+e5m2 / nvfp8). The block-scaled counterpart of -//! the affine `ffai/flash_quantized_sdpa.rs`: K and V are dequantized inline +//! the affine `ffai/mt_flash_quantized_sdpa.rs`: K and V are dequantized inline //! per thread via `element_decode(code) · block_scale` (no bias) instead of the //! affine `q·scale + bias`. //! @@ -1827,7 +1827,7 @@ pub mod kernel_tests { use super::*; use crate::{ - quant::format::QFormat, + kernels::quant::format::QFormat, utils::{pack_f32, unpack_f32}, }; @@ -1915,10 +1915,10 @@ pub mod kernel_tests { let q = unpack_f32(&pack_f32(&source(q_heads * dim, 0x51, 2.0), dt), dt); let k_raw = source(rows * dim, 0x62, 3.0); let v_raw = source(rows * dim, 0x73, 3.0); - let kp = crate::quant::format::pack(fmt, &k_raw, rows, dim); - let vp = crate::quant::format::pack(fmt, &v_raw, rows, dim); - let k_deq = crate::quant::format::dequant(fmt, &kp, rows, dim); - let v_deq = crate::quant::format::dequant(fmt, &vp, rows, dim); + let kp = crate::kernels::quant::format::pack(fmt, &k_raw, rows, dim); + let vp = crate::kernels::quant::format::pack(fmt, &v_raw, rows, dim); + let k_deq = crate::kernels::quant::format::dequant(fmt, &kp, rows, dim); + let v_deq = crate::kernels::quant::format::dequant(fmt, &vp, rows, dim); let sinks: Vec = if has_sinks { (0..q_heads).map(|h| 0.5 + 0.25 * h as f32).collect() } else { @@ -1943,8 +1943,8 @@ pub mod kernel_tests { // off the format so new integer formats pick up the right buffer types. let weight_dt = if fmt.element_bits() == 8 { DType::U8 } else { DType::U32 }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let mut s = TestSetup::new(kernel) @@ -2671,7 +2671,7 @@ pub mod kernel_benches { use metaltile::{bench, core::ir::Kernel, test::*}; use super::*; - use crate::quant::format::QFormat; + use crate::kernels::quant::format::QFormat; fn flash_bench(kernel: Kernel, fmt: QFormat, dim: usize, dt: DType) -> BenchSetup { let (q_heads, kv_heads, tokens) = (8usize, 1usize, 2048usize); @@ -2683,11 +2683,14 @@ pub mod kernel_benches { let (codes_len, codes_dt) = if fmt.element_bits() == 8 { (rows * dim, DType::U8) } else { - (crate::quant::format::bitstream_words(rows * dim, fmt.element_bits()), DType::U32) + ( + crate::kernels::quant::format::bitstream_words(rows * dim, fmt.element_bits()), + DType::U32, + ) }; let scales_dt = match fmt.scale_kind() { - crate::quant::format::ScaleKind::F32 => DType::F32, - crate::quant::format::ScaleKind::F16 => DType::F16, + crate::kernels::quant::format::ScaleKind::F32 => DType::F32, + crate::kernels::quant::format::ScaleKind::F16 => DType::F16, _ => DType::U8, }; let sz = dt.size_bytes(); diff --git a/crates/metaltile-std/src/ffai/flash_quantized_sdpa.rs b/crates/metaltile-std/src/kernels/sdpa/flash_quantized_sdpa.rs similarity index 93% rename from crates/metaltile-std/src/ffai/flash_quantized_sdpa.rs rename to crates/metaltile-std/src/kernels/sdpa/flash_quantized_sdpa.rs index 5fe3fff5..e9f2d075 100644 --- a/crates/metaltile-std/src/ffai/flash_quantized_sdpa.rs +++ b/crates/metaltile-std/src/kernels/sdpa/flash_quantized_sdpa.rs @@ -1,9 +1,9 @@ //! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric //! SPDX-License-Identifier: Apache-2.0 //! Flash quantized SDPA — single-pass online-softmax attention over an -//! affine-quantized K/V cache. Port of `flash_quantized_sdpa.h` +//! affine-quantized K/V cache. Port of `mt_flash_quantized_sdpa.h` //! (spec 041 phase 1.1/1.2). The affine-quant counterpart of -//! `aura_flash_sdpa`: K and V are dequantized inline per thread from +//! `mt_aura_flash_sdpa`: K and V are dequantized inline per thread from //! packed-index + per-group scale + bias triples (the layout //! `quantized` matmul consumes), instead of an AURA codebook. //! @@ -23,7 +23,7 @@ //! //! Lane `program_id::<0>()` ∈ [0,32) owns dim slots `lane + i*32`; //! `program_id::<1>()` = query index. Single-simdgroup shape, matching -//! `aura_flash_sdpa` (token-parallelism is a perf follow-up). +//! `mt_aura_flash_sdpa` (token-parallelism is a perf follow-up). //! //! ## Mask variants //! @@ -31,14 +31,14 @@ //! addition to the built-in causal / sliding-window guard. Two new //! constexpr-gated kernel variants cover the MLX-upstream mask shapes: //! -//! - **Bool mask** (`flash_quantized_sdpa_bool_mask_b{4,8}_d{64,128,256}`): +//! - **Bool mask** (`mt_flash_quantized_sdpa_bool_mask_b{4,8}_d{64,128,256}`): //! takes a `mask_bool: Tensor` of shape `[B*nQ, tokens]` (packed //! as u32, one bit per token) — or flat byte-per-token; see note below. //! When `mask_bool[q_idx * tokens + t] == 0` the key at position `t` //! is skipped (softmax weight set to zero). Useful for segment packing //! and cross-sequence masking. //! -//! - **Float mask** (`flash_quantized_sdpa_float_mask_b{4,8}_d{64,128,256}`): +//! - **Float mask** (`mt_flash_quantized_sdpa_float_mask_b{4,8}_d{64,128,256}`): //! takes a `mask_float: Tensor` of shape `[B*nQ, tokens]`. //! The value `mask_float[q_idx * tokens + t]` is added to the raw //! attention logit before the online-softmax step, enabling relative- @@ -52,7 +52,7 @@ //! The mask buffers are per-element (one f32/T or one u32 per token per //! query), row-major `[B*nQ, tokens]`. For the bool mask, each slot is //! a full `u32` (0 = masked, non-zero = visible) — matching the MLX -//! `mask_t` convention used in `aura_flash_sdpa`. +//! `mask_t` convention used in `mt_aura_flash_sdpa`. //! //! ## DISPATCH INVARIANTS //! @@ -73,7 +73,7 @@ use metaltile::kernel; /// Flash quantized SDPA (no mask), BITS ∈ {4, 8}, DIM ∈ {64,96,128,256,512}. /// -/// Produces 10 kernels: `flash_quantized_sdpa_b4_d64`, `_b4_d96`, `_b4_d128`, +/// Produces 10 kernels: `mt_flash_quantized_sdpa_b4_d64`, `_b4_d96`, `_b4_d128`, /// `_b4_d256`, `_b4_d512`, `_b8_d64`, `_b8_d96`, `_b8_d128`, `_b8_d256`, /// `_b8_d512`. #[kernel(variants( @@ -82,7 +82,7 @@ use metaltile::kernel; DPL = [2, 3, 4, 8, 16, 2, 3, 4, 8, 16], suffix = "b{BITS}_d{DIM}" ))] -pub fn flash_quantized_sdpa( +pub fn mt_flash_quantized_sdpa( queries: Tensor, k_packed: Tensor, k_scales: Tensor, @@ -197,7 +197,7 @@ pub fn flash_quantized_sdpa( /// Flash quantized SDPA with bool gate mask, BITS ∈ {4, 8}, DIM ∈ {64,128,256}. /// -/// Produces 6 kernels: `flash_quantized_sdpa_bool_mask_b4_d64`, +/// Produces 6 kernels: `mt_flash_quantized_sdpa_bool_mask_b4_d64`, /// `_b4_d128`, `_b4_d256`, `_b8_d64`, `_b8_d128`, `_b8_d256`. #[kernel(variants( BITS = [4, 4, 4, 8, 8, 8], @@ -205,7 +205,7 @@ pub fn flash_quantized_sdpa( DPL = [2, 4, 8, 2, 4, 8], suffix = "b{BITS}_d{DIM}" ))] -pub fn flash_quantized_sdpa_bool_mask( +pub fn mt_flash_quantized_sdpa_bool_mask( queries: Tensor, k_packed: Tensor, k_scales: Tensor, @@ -319,7 +319,7 @@ pub fn flash_quantized_sdpa_bool_mask( /// Flash quantized SDPA with additive float bias mask, BITS ∈ {4, 8}, DIM ∈ {64,128,256}. /// -/// Produces 6 kernels: `flash_quantized_sdpa_float_mask_b4_d64`, +/// Produces 6 kernels: `mt_flash_quantized_sdpa_float_mask_b4_d64`, /// `_b4_d128`, `_b4_d256`, `_b8_d64`, `_b8_d128`, `_b8_d256`. #[kernel(variants( BITS = [4, 4, 4, 8, 8, 8], @@ -327,7 +327,7 @@ pub fn flash_quantized_sdpa_bool_mask( DPL = [2, 4, 8, 2, 4, 8], suffix = "b{BITS}_d{DIM}" ))] -pub fn flash_quantized_sdpa_float_mask( +pub fn mt_flash_quantized_sdpa_float_mask( queries: Tensor, k_packed: Tensor, k_scales: Tensor, @@ -439,14 +439,14 @@ pub mod kernel_tests { use metaltile::{test::*, test_kernel}; use super::{ - flash_quantized_sdpa_b4_d96, - flash_quantized_sdpa_b4_d128, - flash_quantized_sdpa_b4_d512, - flash_quantized_sdpa_b8_d128, - flash_quantized_sdpa_bool_mask_b4_d128, - flash_quantized_sdpa_bool_mask_b8_d128, - flash_quantized_sdpa_float_mask_b4_d128, - flash_quantized_sdpa_float_mask_b8_d128, + mt_flash_quantized_sdpa_b4_d96, + mt_flash_quantized_sdpa_b4_d128, + mt_flash_quantized_sdpa_b4_d512, + mt_flash_quantized_sdpa_b8_d128, + mt_flash_quantized_sdpa_bool_mask_b4_d128, + mt_flash_quantized_sdpa_bool_mask_b8_d128, + mt_flash_quantized_sdpa_float_mask_b4_d128, + mt_flash_quantized_sdpa_float_mask_b8_d128, }; use crate::utils::{pack_f32, unpack_f32}; @@ -749,50 +749,78 @@ pub mod kernel_tests { // Base b4_d128, full attention, no sinks. #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_b4_d128(dt: DType) -> TestSetup { - base_setup(flash_quantized_sdpa_b4_d128::kernel_ir_for(dt), 128, 4, 64, false, 0, dt) + base_setup(mt_flash_quantized_sdpa_b4_d128::kernel_ir_for(dt), 128, 4, 64, false, 0, dt) } // Attention sinks. #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_b4_d128_sinks(dt: DType) -> TestSetup { - base_setup(flash_quantized_sdpa_b4_d128::kernel_ir_for(dt), 128, 4, 64, true, 0, dt) + base_setup(mt_flash_quantized_sdpa_b4_d128::kernel_ir_for(dt), 128, 4, 64, true, 0, dt) } // Sliding window (4 of 8 tokens). #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_b4_d128_window(dt: DType) -> TestSetup { - base_setup(flash_quantized_sdpa_b4_d128::kernel_ir_for(dt), 128, 4, 64, false, 4, dt) + base_setup(mt_flash_quantized_sdpa_b4_d128::kernel_ir_for(dt), 128, 4, 64, false, 4, dt) } // 8-bit quant (pack_factor 4). #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_b8_d128(dt: DType) -> TestSetup { - base_setup(flash_quantized_sdpa_b8_d128::kernel_ir_for(dt), 128, 8, 64, false, 0, dt) + base_setup(mt_flash_quantized_sdpa_b8_d128::kernel_ir_for(dt), 128, 8, 64, false, 0, dt) } // head_dim 96 (dims_per_lane 3) and 512 (dims_per_lane 16). #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_b4_d96(dt: DType) -> TestSetup { - base_setup(flash_quantized_sdpa_b4_d96::kernel_ir_for(dt), 96, 4, 32, false, 0, dt) + base_setup(mt_flash_quantized_sdpa_b4_d96::kernel_ir_for(dt), 96, 4, 32, false, 0, dt) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_b4_d512(dt: DType) -> TestSetup { - base_setup(flash_quantized_sdpa_b4_d512::kernel_ir_for(dt), 512, 4, 64, false, 0, dt) + base_setup(mt_flash_quantized_sdpa_b4_d512::kernel_ir_for(dt), 512, 4, 64, false, 0, dt) } // Bool-mask gate (b4 / b8). #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_bool_mask_b4_d128(dt: DType) -> TestSetup { - mask_setup(flash_quantized_sdpa_bool_mask_b4_d128::kernel_ir_for(dt), 128, 4, 64, false, dt) + mask_setup( + mt_flash_quantized_sdpa_bool_mask_b4_d128::kernel_ir_for(dt), + 128, + 4, + 64, + false, + dt, + ) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_bool_mask_b8_d128(dt: DType) -> TestSetup { - mask_setup(flash_quantized_sdpa_bool_mask_b8_d128::kernel_ir_for(dt), 128, 8, 64, false, dt) + mask_setup( + mt_flash_quantized_sdpa_bool_mask_b8_d128::kernel_ir_for(dt), + 128, + 8, + 64, + false, + dt, + ) } // Float-mask additive bias (b4 / b8). #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_float_mask_b4_d128(dt: DType) -> TestSetup { - mask_setup(flash_quantized_sdpa_float_mask_b4_d128::kernel_ir_for(dt), 128, 4, 64, true, dt) + mask_setup( + mt_flash_quantized_sdpa_float_mask_b4_d128::kernel_ir_for(dt), + 128, + 4, + 64, + true, + dt, + ) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-2, 1e-1])] fn test_ffai_flash_quantized_sdpa_float_mask_b8_d128(dt: DType) -> TestSetup { - mask_setup(flash_quantized_sdpa_float_mask_b8_d128::kernel_ir_for(dt), 128, 8, 64, true, dt) + mask_setup( + mt_flash_quantized_sdpa_float_mask_b8_d128::kernel_ir_for(dt), + 128, + 8, + 64, + true, + dt, + ) } } @@ -902,7 +930,7 @@ pub mod kernel_benches { DIM = [64, 96, 128, 256, 512, 64, 96, 128, 256, 512], suffix = "b{BITS}_d{DIM}"))] fn bench_base(dt: DType) -> BenchSetup { - base(flash_quantized_sdpa_bBITS_dDIM::kernel_ir_for(dt), DIM, BITS, dt) + base(mt_flash_quantized_sdpa_bBITS_dDIM::kernel_ir_for(dt), DIM, BITS, dt) } // Bool mask: BITS ∈ {4, 8}, DIM ∈ {64, 128, 256} — 6 combos. @@ -911,7 +939,7 @@ pub mod kernel_benches { suffix = "b{BITS}_d{DIM}"))] fn bench_bool_mask(dt: DType) -> BenchSetup { masked( - flash_quantized_sdpa_bool_mask_bBITS_dDIM::kernel_ir_for(dt), + mt_flash_quantized_sdpa_bool_mask_bBITS_dDIM::kernel_ir_for(dt), DIM, BITS, "mask_bool", @@ -926,7 +954,7 @@ pub mod kernel_benches { suffix = "b{BITS}_d{DIM}"))] fn bench_float_mask(dt: DType) -> BenchSetup { masked( - flash_quantized_sdpa_float_mask_bBITS_dDIM::kernel_ir_for(dt), + mt_flash_quantized_sdpa_float_mask_bBITS_dDIM::kernel_ir_for(dt), DIM, BITS, "mask_float", diff --git a/crates/metaltile-std/src/ffai/dsv4_indexer_score.rs b/crates/metaltile-std/src/kernels/sdpa/indexer_score.rs similarity index 95% rename from crates/metaltile-std/src/ffai/dsv4_indexer_score.rs rename to crates/metaltile-std/src/kernels/sdpa/indexer_score.rs index fe7f9ed0..3769df21 100644 --- a/crates/metaltile-std/src/ffai/dsv4_indexer_score.rs +++ b/crates/metaltile-std/src/kernels/sdpa/indexer_score.rs @@ -19,7 +19,7 @@ //! DSv4 production shape: `n_heads = 64`, `d_idx = 128`. The //! indexer's job is to pick 512 cache positions per CSA / HCA layer //! per decode step; this kernel produces the aggregate score, and a -//! follow-up `ffai_dsv4_indexer_topk` does the bitonic top-512. +//! follow-up `mt_indexer_topk` does the bitonic top-512. //! //! ## Dispatch //! @@ -35,7 +35,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_dsv4_indexer_score( +pub fn mt_indexer_score( q_idx: Tensor, k_idx: Tensor, w: Tensor, @@ -70,7 +70,7 @@ pub fn ffai_dsv4_indexer_score( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_indexer_score; + use super::mt_indexer_score; use crate::utils::{pack_f32, unpack_f32}; fn cpu_reference( @@ -111,7 +111,7 @@ pub mod kernel_tests { let q_dt = unpack_f32(&pack_f32(&q_idx, dt), dt); let k_dt = unpack_f32(&pack_f32(&k_idx, dt), dt); let expected = cpu_reference(&q_dt, &k_dt, &w, n_heads, d_idx, n_kv); - TestSetup::new(ffai_dsv4_indexer_score::kernel_ir_for(dt)) + TestSetup::new(mt_indexer_score::kernel_ir_for(dt)) .input(TestBuffer::from_vec("q_idx", pack_f32(&q_idx, dt), dt)) .input(TestBuffer::from_vec("k_idx", pack_f32(&k_idx, dt), dt)) .input(TestBuffer::from_vec("w", pack_f32(&w, DType::F32), DType::F32)) @@ -136,14 +136,14 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_indexer_score; + use super::mt_indexer_score; #[bench(dtypes = [f32, f16, bf16])] fn bench_indexer(dt: DType) -> BenchSetup { let (n_heads, d_idx, n_kv) = (64usize, 128usize, 4096usize); let bytes = (n_heads * d_idx + n_kv * n_heads * d_idx) * dt.size_bytes() + n_heads * 4 + n_kv * 4; - BenchSetup::new(ffai_dsv4_indexer_score::kernel_ir_for(dt)) + BenchSetup::new(mt_indexer_score::kernel_ir_for(dt)) .buffer(BenchBuffer::random("q_idx", n_heads * d_idx, dt)) .buffer(BenchBuffer::random("k_idx", n_kv * n_heads * d_idx, dt)) .buffer(BenchBuffer::random("w", n_heads, DType::F32)) diff --git a/crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs b/crates/metaltile-std/src/kernels/sdpa/indexer_topk.rs similarity index 95% rename from crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs rename to crates/metaltile-std/src/kernels/sdpa/indexer_topk.rs index 838aa33b..5990712f 100644 --- a/crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs +++ b/crates/metaltile-std/src/kernels/sdpa/indexer_topk.rs @@ -3,7 +3,7 @@ //! DSv4 Lightning Indexer — top-k index selection over per-position //! aggregate scores. //! -//! Sits downstream of [`ffai_dsv4_indexer_score`]. Takes the +//! Sits downstream of [`mt_indexer_score`]. Takes the //! `score[n_kv]` produced by the indexer score kernel and returns the //! indices of the K largest entries (DSv4 production: `K = 512`). The //! returned indices feed CSA's sparse-gather SDPA inner loop. @@ -34,7 +34,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_dsv4_indexer_topk_block( +pub fn mt_indexer_topk_block( score: Tensor, mut out_indices: Tensor, #[constexpr] n_kv: u32, @@ -105,7 +105,7 @@ pub fn ffai_dsv4_indexer_topk_block( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_dsv4_indexer_topk_block; + use super::mt_indexer_topk_block; use crate::utils::pack_f32; /// CPU reference: argsort scores descending, take first `k` @@ -128,7 +128,7 @@ pub mod kernel_tests { // Pack u32 expected as raw little-endian bytes for the test // framework — same path mxfp4_dequant uses for u32 outputs. let expected_bytes: Vec = expected.iter().flat_map(|i| i.to_le_bytes()).collect(); - TestSetup::new(ffai_dsv4_indexer_topk_block::kernel_ir()) + TestSetup::new(mt_indexer_topk_block::kernel_ir()) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("score", pack_f32(&scores, DType::F32), DType::F32)) .input(TestBuffer::zeros("out_indices", k, DType::U32)) @@ -154,12 +154,12 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_dsv4_indexer_topk_block; + use super::mt_indexer_topk_block; #[bench(dtypes = [f32])] fn bench_topk(_dt: DType) -> BenchSetup { let (n_kv, k) = (1024usize, 512usize); - BenchSetup::new(ffai_dsv4_indexer_topk_block::kernel_ir()) + BenchSetup::new(mt_indexer_topk_block::kernel_ir()) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("score", n_kv, DType::F32)) .buffer(BenchBuffer::zeros("out_indices", k, DType::U32).output()) diff --git a/crates/metaltile-std/src/kernels/sdpa/mod.rs b/crates/metaltile-std/src/kernels/sdpa/mod.rs new file mode 100644 index 00000000..49ad2d52 --- /dev/null +++ b/crates/metaltile-std/src/kernels/sdpa/mod.rs @@ -0,0 +1,36 @@ +//! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric +//! SPDX-License-Identifier: Apache-2.0 +//! Scaled-dot-product attention — the sdpa family (see +//! `docs/specs/KERNEL_CONSOLIDATION_PLAN.md`): the bidirectional / decode / +//! multi / prefill paths across head-dims (d64..d512) and patterns (relpos, +//! windowed, conformer, 2-pass, batched, sink), the quantized + block-scaled +//! flash forms, the AURA compressed-domain attention, and the DSv4 compressed +//! sparse-attention (CSA) decode + Lightning Indexer. Migrated from `mlx/` + +//! `ffai/`; model names dropped (dsv4 → csa/indexer/compressor). + +pub mod attn_head_gate; +pub mod aura_flash_p1; +pub mod aura_flash_pass2; +pub mod aura_flash_sdpa; +pub mod aura_score; +pub mod aura_value; +pub mod compressor_pool; +pub mod csa_sdpa_decode; +pub mod flash_block_scaled_sdpa; +pub mod flash_quantized_sdpa; +pub mod indexer_score; +pub mod indexer_topk; +pub mod scaled_dot_product_attention; +pub mod sdpa_bidirectional; +pub mod sdpa_bidirectional_d128_relpos; +pub mod sdpa_bidirectional_windowed; +pub mod sdpa_decode; +pub mod sdpa_decode_2pass; +pub mod sdpa_decode_batched; +pub mod sdpa_decode_d512_sink; +pub mod sdpa_decode_sink_buf; +pub mod sdpa_multi; +pub mod sdpa_multi_d256; +pub mod sdpa_prefill_d512_sink; +pub mod sdpa_rel_pos_conformer; +pub mod sdpa_vector; diff --git a/crates/metaltile-std/src/mlx/scaled_dot_product_attention.rs b/crates/metaltile-std/src/kernels/sdpa/scaled_dot_product_attention.rs similarity index 99% rename from crates/metaltile-std/src/mlx/scaled_dot_product_attention.rs rename to crates/metaltile-std/src/kernels/sdpa/scaled_dot_product_attention.rs index 6442909d..02b498b8 100644 --- a/crates/metaltile-std/src/mlx/scaled_dot_product_attention.rs +++ b/crates/metaltile-std/src/kernels/sdpa/scaled_dot_product_attention.rs @@ -109,7 +109,7 @@ pub mod kernel_tests { use super::mt_sdpa; use crate::{ - mlx::sdpa_vector::kernel_tests::cpu_sdpa, + kernels::sdpa::sdpa_vector::kernel_tests::cpu_sdpa, utils::{pack_f32, unpack_f32}, }; diff --git a/crates/metaltile-std/src/ffai/sdpa_bidirectional.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional.rs similarity index 95% rename from crates/metaltile-std/src/ffai/sdpa_bidirectional.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional.rs index 8b0ab419..3beba5a2 100644 --- a/crates/metaltile-std/src/ffai/sdpa_bidirectional.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional.rs @@ -15,7 +15,7 @@ //! //! ## Naming //! -//! `ffai_sdpa_bidirectional_dN` — N is the constexpr head_dim, +//! `mt_sdpa_bidirectional_dN` — N is the constexpr head_dim, //! T is the element type (f32 / f16 / bf16). //! //! Two per-lane layout families: @@ -33,7 +33,7 @@ //! //! Reduction-mode kernels — STRICT threadgroup geometry. Wrappers MUST //! encode these as preconditions; the same machine-freeze hazard as -//! `ffai_sdpa_decode` / `ffai_sdpa_multi`. +//! `mt_sdpa_decode` / `mt_sdpa_multi`. //! //! - **TPG = 1024 threads** (32 simdgroups × 32 lanes). Hard. A TPG //! below 32 makes `n_simd = TPG / 32 = 0`, turning the K walk @@ -59,7 +59,7 @@ use metaltile::kernel; // ─── head_dim = 64 (SigLIP base/large, CLIP-L) ───────────────────── #[kernel] -pub fn ffai_sdpa_bidirectional_d64( +pub fn mt_sdpa_bidirectional_d64( q: Tensor, k: Tensor, v: Tensor, @@ -165,7 +165,7 @@ pub fn ffai_sdpa_bidirectional_d64( // ─── head_dim = 32 (FastViT-HD) ──────────────────────────────────── #[kernel] -pub fn ffai_sdpa_bidirectional_d32( +pub fn mt_sdpa_bidirectional_d32( q: Tensor, k: Tensor, v: Tensor, @@ -261,7 +261,7 @@ pub fn ffai_sdpa_bidirectional_d32( // a 32-wide simdgroup. #[kernel] -pub fn ffai_sdpa_bidirectional_d72( +pub fn mt_sdpa_bidirectional_d72( q: Tensor, k: Tensor, v: Tensor, @@ -394,7 +394,7 @@ pub fn ffai_sdpa_bidirectional_d72( // the same per-element bounds masks. #[kernel] -pub fn ffai_sdpa_bidirectional_d80( +pub fn mt_sdpa_bidirectional_d80( q: Tensor, k: Tensor, v: Tensor, @@ -512,7 +512,7 @@ pub fn ffai_sdpa_bidirectional_d80( // 24-active-lane geometry; here every lane participates. #[kernel] -pub fn ffai_sdpa_bidirectional_d96( +pub fn mt_sdpa_bidirectional_d96( q: Tensor, k: Tensor, v: Tensor, @@ -619,11 +619,11 @@ pub mod kernel_tests { use metaltile::{test::*, test_kernel}; use super::{ - ffai_sdpa_bidirectional_d32, - ffai_sdpa_bidirectional_d64, - ffai_sdpa_bidirectional_d72, - ffai_sdpa_bidirectional_d80, - ffai_sdpa_bidirectional_d96, + mt_sdpa_bidirectional_d32, + mt_sdpa_bidirectional_d64, + mt_sdpa_bidirectional_d72, + mt_sdpa_bidirectional_d80, + mt_sdpa_bidirectional_d96, }; use crate::utils::{pack_f32, unpack_f32}; @@ -720,12 +720,12 @@ pub mod kernel_tests { // head_dim 32 — exactly one element per lane (32 / 32 = 1). #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_ffai_sdpa_bidirectional_d32(dt: DType) -> TestSetup { - setup(ffai_sdpa_bidirectional_d32::kernel_ir_for(dt), 32, dt) + setup(mt_sdpa_bidirectional_d32::kernel_ir_for(dt), 32, dt) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_ffai_sdpa_bidirectional_d64(dt: DType) -> TestSetup { - setup(ffai_sdpa_bidirectional_d64::kernel_ir_for(dt), 64, dt) + setup(mt_sdpa_bidirectional_d64::kernel_ir_for(dt), 64, dt) } // head_dim 72 — ragged: 72 = 32·2 + 8, so the first 8 lanes own a third @@ -733,18 +733,18 @@ pub mod kernel_tests { // masking the clean-fit dims leave dormant. #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_ffai_sdpa_bidirectional_d72(dt: DType) -> TestSetup { - setup(ffai_sdpa_bidirectional_d72::kernel_ir_for(dt), 72, dt) + setup(mt_sdpa_bidirectional_d72::kernel_ir_for(dt), 72, dt) } // head_dim 80 — ragged: 80 = 32·2 + 16, partial-lane bounds masking. #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_ffai_sdpa_bidirectional_d80(dt: DType) -> TestSetup { - setup(ffai_sdpa_bidirectional_d80::kernel_ir_for(dt), 80, dt) + setup(mt_sdpa_bidirectional_d80::kernel_ir_for(dt), 80, dt) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_ffai_sdpa_bidirectional_d96(dt: DType) -> TestSetup { - setup(ffai_sdpa_bidirectional_d96::kernel_ir_for(dt), 96, dt) + setup(mt_sdpa_bidirectional_d96::kernel_ir_for(dt), 96, dt) } // Production vision-tower self-attention shape that previously @@ -774,7 +774,7 @@ pub mod kernel_tests { &q, &k, &v, n_q_heads, n_kv_heads, head_dim, base_kv, n_query, kv_stride, scale, ); - TestSetup::new(ffai_sdpa_bidirectional_d64::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_bidirectional_d64::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -802,11 +802,11 @@ pub mod kernel_benches { use metaltile::{bench, test::*}; use super::{ - ffai_sdpa_bidirectional_d32, - ffai_sdpa_bidirectional_d64, - ffai_sdpa_bidirectional_d72, - ffai_sdpa_bidirectional_d80, - ffai_sdpa_bidirectional_d96, + mt_sdpa_bidirectional_d32, + mt_sdpa_bidirectional_d64, + mt_sdpa_bidirectional_d72, + mt_sdpa_bidirectional_d80, + mt_sdpa_bidirectional_d96, }; fn setup(ir: metaltile::core::ir::Kernel, head_dim: usize, dt: DType) -> BenchSetup { @@ -839,26 +839,26 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16])] fn bench_d32(dt: DType) -> BenchSetup { - setup(ffai_sdpa_bidirectional_d32::kernel_ir_for(dt), 32, dt) + setup(mt_sdpa_bidirectional_d32::kernel_ir_for(dt), 32, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_d64(dt: DType) -> BenchSetup { - setup(ffai_sdpa_bidirectional_d64::kernel_ir_for(dt), 64, dt) + setup(mt_sdpa_bidirectional_d64::kernel_ir_for(dt), 64, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_d72(dt: DType) -> BenchSetup { - setup(ffai_sdpa_bidirectional_d72::kernel_ir_for(dt), 72, dt) + setup(mt_sdpa_bidirectional_d72::kernel_ir_for(dt), 72, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_d80(dt: DType) -> BenchSetup { - setup(ffai_sdpa_bidirectional_d80::kernel_ir_for(dt), 80, dt) + setup(mt_sdpa_bidirectional_d80::kernel_ir_for(dt), 80, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_d96(dt: DType) -> BenchSetup { - setup(ffai_sdpa_bidirectional_d96::kernel_ir_for(dt), 96, dt) + setup(mt_sdpa_bidirectional_d96::kernel_ir_for(dt), 96, dt) } } diff --git a/crates/metaltile-std/src/ffai/sdpa_bidirectional_d128_relpos.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional_d128_relpos.rs similarity index 97% rename from crates/metaltile-std/src/ffai/sdpa_bidirectional_d128_relpos.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional_d128_relpos.rs index 4a2acf2b..adeef43e 100644 --- a/crates/metaltile-std/src/ffai/sdpa_bidirectional_d128_relpos.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional_d128_relpos.rs @@ -40,7 +40,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_bidirectional_d128_relpos( +pub fn mt_sdpa_bidirectional_d128_relpos( q: Tensor, k: Tensor, v: Tensor, @@ -177,7 +177,7 @@ pub fn ffai_sdpa_bidirectional_d128_relpos( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_bidirectional_d128_relpos; + use super::mt_sdpa_bidirectional_d128_relpos; use crate::utils::{pack_f32, unpack_f32}; // Per (query, q_head): softmax(Q·Kᵀ·scale + relpos_bias)·V over @@ -267,7 +267,7 @@ pub mod kernel_tests { rel_zero, rel_len, scale, ); - TestSetup::new(ffai_sdpa_bidirectional_d128_relpos::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_bidirectional_d128_relpos::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -296,7 +296,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_bidirectional_d128_relpos; + use super::mt_sdpa_bidirectional_d128_relpos; #[bench(dtypes = [f32, f16, bf16])] fn bench_d128_relpos(dt: DType) -> BenchSetup { @@ -311,7 +311,7 @@ pub mod kernel_benches { let n_kv = base_kv + n_query; let bytes = (2 * n_query * n_q_heads * head_dim + 2 * n_kv_heads * n_kv * head_dim) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_bidirectional_d128_relpos::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_bidirectional_d128_relpos::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_bidirectional_windowed.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional_windowed.rs similarity index 97% rename from crates/metaltile-std/src/ffai/sdpa_bidirectional_windowed.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional_windowed.rs index c24bf296..2acce492 100644 --- a/crates/metaltile-std/src/ffai/sdpa_bidirectional_windowed.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_bidirectional_windowed.rs @@ -17,7 +17,7 @@ //! //! `head_dim == 80` (Qwen2.5-VL: hidden 1280 / 16 heads). Ragged 3-element //! layout (lanes 0..26 own indices 0..79, lane 26's 3rd element and lanes -//! 27..31 are bounds-masked) — identical to `ffai_sdpa_bidirectional_d80`. +//! 27..31 are bounds-masked) — identical to `mt_sdpa_bidirectional_d80`. //! //! ## Window contract //! @@ -38,7 +38,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_bidirectional_windowed_d80( +pub fn mt_sdpa_bidirectional_windowed_d80( q: Tensor, k: Tensor, v: Tensor, @@ -156,7 +156,7 @@ pub fn ffai_sdpa_bidirectional_windowed_d80( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_bidirectional_windowed_d80; + use super::mt_sdpa_bidirectional_windowed_d80; use crate::utils::{pack_f32, unpack_f32}; // Per (query, q_head): softmax(scale·Q·Kᵀ)·V over THIS query's window @@ -249,7 +249,7 @@ pub mod kernel_tests { scale, ); - TestSetup::new(ffai_sdpa_bidirectional_windowed_d80::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_bidirectional_windowed_d80::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -289,7 +289,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_bidirectional_windowed_d80; + use super::mt_sdpa_bidirectional_windowed_d80; #[bench(dtypes = [f32, f16, bf16])] fn bench_windowed_d80(dt: DType) -> BenchSetup { @@ -309,7 +309,7 @@ pub mod kernel_benches { } let bytes = (2 * n_query * n_q_heads * head_dim + 2 * n_kv_heads * n_query * head_dim) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_bidirectional_windowed_d80::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_bidirectional_windowed_d80::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_decode.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode.rs similarity index 95% rename from crates/metaltile-std/src/ffai/sdpa_decode.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_decode.rs index 1311384f..25c89241 100644 --- a/crates/metaltile-std/src/ffai/sdpa_decode.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode.rs @@ -4,11 +4,11 @@ //! //! | Kernel | elems | Phases | Sink | Window | //! |-------------------------|-------|--------|------|--------| -//! | `ffai_sdpa_decode_d64` | 2 | 1×2 | ✓ | | -//! | `ffai_sdpa_decode_d96` | 3 | 1×3 | | | -//! | `ffai_sdpa_decode` | 4 | 1×4 | ✓ | ✓ | -//! | `ffai_sdpa_decode_d256` | 8 | 2×4 | ✓ | | -//! | `ffai_sdpa_decode_d512` | 16 | 4×4 | | | +//! | `mt_sdpa_decode_d64` | 2 | 1×2 | ✓ | | +//! | `mt_sdpa_decode_d96` | 3 | 1×3 | | | +//! | `mt_sdpa_decode` | 4 | 1×4 | ✓ | ✓ | +//! | `mt_sdpa_decode_d256` | 8 | 2×4 | ✓ | | +//! | `mt_sdpa_decode_d512` | 16 | 4×4 | | | //! //! Each inner `range(0u32, N, 1u32)` loop uses a literal bound so the //! DSL's `UnrollPass` unrolls it to constant-indexed `stack_alloc` @@ -25,7 +25,7 @@ use metaltile::kernel; // ── d64: 2-slot single phase, sink-aware reduction ────────────────────── #[kernel] -pub fn ffai_sdpa_decode_d64( +pub fn mt_sdpa_decode_d64( q: Tensor, k: Tensor, v: Tensor, @@ -127,7 +127,7 @@ pub fn ffai_sdpa_decode_d64( // ── d96: 3-slot single phase, simple reduction ────────────────────────── #[kernel] -pub fn ffai_sdpa_decode_d96( +pub fn mt_sdpa_decode_d96( q: Tensor, k: Tensor, v: Tensor, @@ -231,7 +231,7 @@ pub fn ffai_sdpa_decode_d96( // Dense path: sink_end=0, window_start=0 → only second pass fires. #[kernel] -pub fn ffai_sdpa_decode( +pub fn mt_sdpa_decode( q: Tensor, k: Tensor, v: Tensor, @@ -368,7 +368,7 @@ pub fn ffai_sdpa_decode( // ── d256: 4-slot × 2 phases, sink-aware reduction ─────────────────────── #[kernel] -pub fn ffai_sdpa_decode_d256( +pub fn mt_sdpa_decode_d256( q: Tensor, k: Tensor, v: Tensor, @@ -509,7 +509,7 @@ pub fn ffai_sdpa_decode_d256( // 1024-thread pipeline cap, so we use 512 threads. #[kernel] -pub fn ffai_sdpa_decode_d512( +pub fn mt_sdpa_decode_d512( q: Tensor, k: Tensor, v: Tensor, @@ -698,11 +698,11 @@ mod tests { }; use super::{ - ffai_sdpa_decode, - ffai_sdpa_decode_d64, - ffai_sdpa_decode_d96, - ffai_sdpa_decode_d256, - ffai_sdpa_decode_d512, + mt_sdpa_decode, + mt_sdpa_decode_d64, + mt_sdpa_decode_d96, + mt_sdpa_decode_d256, + mt_sdpa_decode_d512, }; fn check(name: &str, src: &str) { @@ -713,30 +713,30 @@ mod tests { #[test] fn codegen_d64() { for dt in [DType::F32, DType::F16, DType::BF16] { - let mut k = ffai_sdpa_decode_d64::kernel_ir_for(dt); + let mut k = mt_sdpa_decode_d64::kernel_ir_for(dt); k.mode = KernelMode::Reduction; let src = MslGenerator::default().generate(&k).unwrap(); - check("ffai_sdpa_decode_d64", &src); + check("mt_sdpa_decode_d64", &src); } } #[test] fn codegen_d96() { for dt in [DType::F32, DType::F16, DType::BF16] { - let mut k = ffai_sdpa_decode_d96::kernel_ir_for(dt); + let mut k = mt_sdpa_decode_d96::kernel_ir_for(dt); k.mode = KernelMode::Reduction; let src = MslGenerator::default().generate(&k).unwrap(); - check("ffai_sdpa_decode_d96", &src); + check("mt_sdpa_decode_d96", &src); } } #[test] fn codegen_d128() { for dt in [DType::F32, DType::F16, DType::BF16] { - let mut k = ffai_sdpa_decode::kernel_ir_for(dt); + let mut k = mt_sdpa_decode::kernel_ir_for(dt); k.mode = KernelMode::Reduction; let src = MslGenerator::default().generate(&k).unwrap(); - check("ffai_sdpa_decode", &src); + check("mt_sdpa_decode", &src); for tok in &["simd_group", "simd_lane", "threadgroup_barrier", "simd_sum", "simd_max"] { assert!(src.contains(tok), "d128 MSL missing `{tok}`:\n{src}"); } @@ -746,20 +746,20 @@ mod tests { #[test] fn codegen_d256() { for dt in [DType::F32, DType::F16, DType::BF16] { - let mut k = ffai_sdpa_decode_d256::kernel_ir_for(dt); + let mut k = mt_sdpa_decode_d256::kernel_ir_for(dt); k.mode = KernelMode::Reduction; let src = MslGenerator::default().generate(&k).unwrap(); - check("ffai_sdpa_decode_d256", &src); + check("mt_sdpa_decode_d256", &src); } } #[test] fn codegen_d512() { for dt in [DType::F32, DType::F16, DType::BF16] { - let mut k = ffai_sdpa_decode_d512::kernel_ir_for(dt); + let mut k = mt_sdpa_decode_d512::kernel_ir_for(dt); k.mode = KernelMode::Reduction; let src = MslGenerator::default().generate(&k).unwrap(); - check("ffai_sdpa_decode_d512", &src); + check("mt_sdpa_decode_d512", &src); } } } @@ -770,11 +770,11 @@ pub mod kernel_tests { use metaltile::{test::*, test_kernel}; use super::{ - ffai_sdpa_decode, - ffai_sdpa_decode_d64, - ffai_sdpa_decode_d96, - ffai_sdpa_decode_d256, - ffai_sdpa_decode_d512, + mt_sdpa_decode, + mt_sdpa_decode_d64, + mt_sdpa_decode_d96, + mt_sdpa_decode_d256, + mt_sdpa_decode_d512, }; use crate::utils::{pack_f32, unpack_f32}; @@ -848,7 +848,7 @@ pub mod kernel_tests { let expected = naive_sdpa( &q, &k, &v, nqh, nkh, hd, n_kv, kv_stride, 0, 0, has_sink, sink_logit, scale, ); - TestSetup::new(ffai_sdpa_decode_d64::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_d64::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -884,7 +884,7 @@ pub mod kernel_tests { let v = unpack_f32(&pack_f32(&ramp(nkh * kv_stride * hd, 0.007, -0.3), dt), dt); let expected = naive_sdpa(&q, &k, &v, nqh, nkh, hd, n_kv, kv_stride, 0, 0, false, 0.0, scale); - TestSetup::new(ffai_sdpa_decode_d96::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_d96::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -932,7 +932,7 @@ pub mod kernel_tests { sink_logit, scale, ); - TestSetup::new(ffai_sdpa_decode::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -995,7 +995,7 @@ pub mod kernel_tests { let expected = naive_sdpa( &q, &k, &v, nqh, nkh, hd, n_kv, kv_stride, 0, 0, has_sink, sink_logit, scale, ); - TestSetup::new(ffai_sdpa_decode_d256::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_d256::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -1031,7 +1031,7 @@ pub mod kernel_tests { let v = unpack_f32(&pack_f32(&ramp(nkh * kv_stride * hd, 0.007, -0.3), dt), dt); let expected = naive_sdpa(&q, &k, &v, nqh, nkh, hd, n_kv, kv_stride, 0, 0, false, 0.0, scale); - TestSetup::new(ffai_sdpa_decode_d512::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_d512::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -1053,11 +1053,11 @@ pub mod kernel_benches { use metaltile::{bench, test::*}; use super::{ - ffai_sdpa_decode, - ffai_sdpa_decode_d64, - ffai_sdpa_decode_d96, - ffai_sdpa_decode_d256, - ffai_sdpa_decode_d512, + mt_sdpa_decode, + mt_sdpa_decode_d64, + mt_sdpa_decode_d96, + mt_sdpa_decode_d256, + mt_sdpa_decode_d512, }; #[bench(dtypes = [f32, f16, bf16])] @@ -1065,7 +1065,7 @@ pub mod kernel_benches { let (nqh, nkh, hd) = (32usize, 8usize, 64usize); let (n_kv, kv_stride) = (4096usize, 4096usize); let bytes = (2 * nqh * hd + 2 * nkh * n_kv * hd) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_decode_d64::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode_d64::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", nqh * hd, dt)) .buffer(BenchBuffer::random("k", nkh * kv_stride * hd, dt)) @@ -1088,7 +1088,7 @@ pub mod kernel_benches { let (nqh, nkh, hd) = (32usize, 8usize, 96usize); let (n_kv, kv_stride) = (4096usize, 4096usize); let bytes = (2 * nqh * hd + 2 * nkh * n_kv * hd) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_decode_d96::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode_d96::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", nqh * hd, dt)) .buffer(BenchBuffer::random("k", nkh * kv_stride * hd, dt)) @@ -1109,7 +1109,7 @@ pub mod kernel_benches { let (nqh, nkh, hd) = (32usize, 8usize, 128usize); let (n_kv, kv_stride) = (4096usize, 4096usize); let bytes = (nqh * hd + 2 * nkh * n_kv * hd + nqh * hd) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_decode::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", nqh * hd, dt)) .buffer(BenchBuffer::random("k", nkh * kv_stride * hd, dt)) @@ -1134,7 +1134,7 @@ pub mod kernel_benches { let (nqh, nkh, hd) = (32usize, 8usize, 256usize); let (n_kv, kv_stride) = (4096usize, 4096usize); let bytes = (2 * nqh * hd + 2 * nkh * n_kv * hd) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_decode_d256::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode_d256::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", nqh * hd, dt)) .buffer(BenchBuffer::random("k", nkh * kv_stride * hd, dt)) @@ -1157,7 +1157,7 @@ pub mod kernel_benches { let (nqh, nkh, hd) = (32usize, 8usize, 512usize); let (n_kv, kv_stride) = (4096usize, 4096usize); let bytes = (2 * nqh * hd + 2 * nkh * n_kv * hd) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_decode_d512::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode_d512::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", nqh * hd, dt)) .buffer(BenchBuffer::random("k", nkh * kv_stride * hd, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_decode_2pass.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_2pass.rs similarity index 97% rename from crates/metaltile-std/src/ffai/sdpa_decode_2pass.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_decode_2pass.rs index aee4532e..33b48afc 100644 --- a/crates/metaltile-std/src/ffai/sdpa_decode_2pass.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_2pass.rs @@ -81,7 +81,7 @@ mod heuristic_tests { // ── Pass 1: per-block partials, GQA co-load ────────────────────────────── #[kernel] -pub fn sdpa_decode_2pass_pass1( +pub fn mt_sdpa_decode_2pass_pass1( q: Tensor, k: Tensor, v: Tensor, @@ -206,7 +206,7 @@ pub fn sdpa_decode_2pass_pass1( // n_kv/blocks is not divisible by 4. #[kernel] -pub fn sdpa_decode_2pass_pass1_bc4( +pub fn mt_sdpa_decode_2pass_pass1_bc4( q: Tensor, k: Tensor, v: Tensor, @@ -465,7 +465,7 @@ pub fn sdpa_decode_2pass_pass1_bc4( // ── Pass 2: 32-sg × 32-lane merge, MLX-style ───────────────────────────── #[kernel] -pub fn sdpa_decode_2pass_pass2( +pub fn mt_sdpa_decode_2pass_pass2( partial_o: Tensor, partial_m: Tensor, partial_l: Tensor, @@ -555,7 +555,7 @@ pub fn sdpa_decode_2pass_pass2( // ── Pass 1 TILED: contiguous-chunk access pattern ──────────────────────── // -// The strided access pattern in `sdpa_decode_2pass_pass1` (each block reads +// The strided access pattern in `mt_sdpa_decode_2pass_pass1` (each block reads // every N-th position: 0, N, 2N, ...) causes L2 cache thrashing on CUDA: // all 512 concurrent TGs are reading from non-overlapping strided positions // in the same KV head, so every load misses L2. Effective bandwidth ~40%. @@ -579,7 +579,7 @@ pub fn sdpa_decode_2pass_pass2( // - For n_kv not divisible by blocks, the last block processes the remainder. #[kernel] -pub fn sdpa_decode_2pass_pass1_tiled( +pub fn mt_sdpa_decode_2pass_pass1_tiled( q: Tensor, k: Tensor, v: Tensor, @@ -685,7 +685,7 @@ pub fn sdpa_decode_2pass_pass1_tiled( /// Pass 1 for head_dim=64. Each lane owns 2 elements (`64/32`). /// Grid: `[n_kv_heads, blocks, 1]`, TG: `[32, gqa_factor, 1]`. #[kernel] -pub fn sdpa_decode_2pass_pass1_d64( +pub fn mt_sdpa_decode_2pass_pass1_d64( q: Tensor, k: Tensor, v: Tensor, @@ -745,7 +745,7 @@ pub fn sdpa_decode_2pass_pass1_d64( /// Pass 2 for head_dim=64. Reduce across `blocks` partials. /// Grid: `[n_q_heads, 1, 1]`, TG: `[1024, 1, 1]`. #[kernel] -pub fn sdpa_decode_2pass_pass2_d64( +pub fn mt_sdpa_decode_2pass_pass2_d64( partial_o: Tensor, partial_m: Tensor, partial_l: Tensor, @@ -803,7 +803,7 @@ pub fn sdpa_decode_2pass_pass2_d64( /// Pass 1 for head_dim=96. Each lane owns 3 elements (`96/32`). #[kernel] -pub fn sdpa_decode_2pass_pass1_d96( +pub fn mt_sdpa_decode_2pass_pass1_d96( q: Tensor, k: Tensor, v: Tensor, @@ -870,7 +870,7 @@ pub fn sdpa_decode_2pass_pass1_d96( /// Pass 2 for head_dim=96. Three tg_out buffers, one per element. #[kernel] -pub fn sdpa_decode_2pass_pass2_d96( +pub fn mt_sdpa_decode_2pass_pass2_d96( partial_o: Tensor, partial_m: Tensor, partial_l: Tensor, @@ -935,7 +935,7 @@ pub fn sdpa_decode_2pass_pass2_d96( /// Pass 1 for head_dim=256. Each lane owns 8 elements (`256/32`). #[kernel] -pub fn sdpa_decode_2pass_pass1_d256( +pub fn mt_sdpa_decode_2pass_pass1_d256( q: Tensor, k: Tensor, v: Tensor, @@ -1042,9 +1042,9 @@ pub fn sdpa_decode_2pass_pass1_d256( } /// Pass 2 for head_dim=256. Four tg_out buffers, reused across two phases -/// (dims 0..3 then 4..7) — same strategy as `ffai_sdpa_decode_d256`. +/// (dims 0..3 then 4..7) — same strategy as `mt_sdpa_decode_d256`. #[kernel] -pub fn sdpa_decode_2pass_pass2_d256( +pub fn mt_sdpa_decode_2pass_pass2_d256( partial_o: Tensor, partial_m: Tensor, partial_l: Tensor, @@ -1151,7 +1151,7 @@ pub fn sdpa_decode_2pass_pass2_d256( // FFAI consumes the pass2 wrapper to chain pass1 → pass2 in // `Ops.sdpaDecode2Pass`. // -// Note: only the base `sdpa_decode_2pass_pass2` kernel gets a +// Note: only the base `mt_sdpa_decode_2pass_pass2` kernel gets a // dedicated emit registration — the d{64,96,256} pass2 siblings // added in PR #157 are reachable through the same Reduction-mode // codegen path (they share the kernel body shape) and only the @@ -1162,7 +1162,7 @@ pub fn sdpa_decode_2pass_pass2_d256( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::sdpa_decode_2pass_pass2; + use super::mt_sdpa_decode_2pass_pass2; use crate::utils::{pack_f32, unpack_f32}; // ── COMBINED two-pass correctness, exercised through pass 2 ────────── @@ -1308,7 +1308,7 @@ pub mod kernel_tests { let expected = naive_sdpa(&q, &k, &v, n_q_heads, gqa_factor, head_dim, n_kv, kv_stride, scale); - TestSetup::new(sdpa_decode_2pass_pass2::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_2pass_pass2::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("partial_o", pack_f32(&partial_o, dt), dt)) .input(TestBuffer::from_vec("partial_m", pack_f32(&partial_m, DType::F32), DType::F32)) @@ -1351,14 +1351,14 @@ pub mod kernel_benches { use metaltile::{bench, test::*}; use super::{ - sdpa_decode_2pass_pass1, - sdpa_decode_2pass_pass1_d64, - sdpa_decode_2pass_pass1_d96, - sdpa_decode_2pass_pass1_d256, - sdpa_decode_2pass_pass2, - sdpa_decode_2pass_pass2_d64, - sdpa_decode_2pass_pass2_d96, - sdpa_decode_2pass_pass2_d256, + mt_sdpa_decode_2pass_pass1, + mt_sdpa_decode_2pass_pass1_d64, + mt_sdpa_decode_2pass_pass1_d96, + mt_sdpa_decode_2pass_pass1_d256, + mt_sdpa_decode_2pass_pass2, + mt_sdpa_decode_2pass_pass2_d64, + mt_sdpa_decode_2pass_pass2_d96, + mt_sdpa_decode_2pass_pass2_d256, }; // Shared decode shape (Qwen3-class GQA, 4096 context, 32-block 2-pass). @@ -1413,41 +1413,41 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16])] fn bench_pass1(dt: DType) -> BenchSetup { - pass1(sdpa_decode_2pass_pass1::kernel_ir_for(dt), 128, dt) + pass1(mt_sdpa_decode_2pass_pass1::kernel_ir_for(dt), 128, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass2(dt: DType) -> BenchSetup { - pass2(sdpa_decode_2pass_pass2::kernel_ir_for(dt), 128, dt) + pass2(mt_sdpa_decode_2pass_pass2::kernel_ir_for(dt), 128, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass1_d64(dt: DType) -> BenchSetup { - pass1(sdpa_decode_2pass_pass1_d64::kernel_ir_for(dt), 64, dt) + pass1(mt_sdpa_decode_2pass_pass1_d64::kernel_ir_for(dt), 64, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass2_d64(dt: DType) -> BenchSetup { - pass2(sdpa_decode_2pass_pass2_d64::kernel_ir_for(dt), 64, dt) + pass2(mt_sdpa_decode_2pass_pass2_d64::kernel_ir_for(dt), 64, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass1_d96(dt: DType) -> BenchSetup { - pass1(sdpa_decode_2pass_pass1_d96::kernel_ir_for(dt), 96, dt) + pass1(mt_sdpa_decode_2pass_pass1_d96::kernel_ir_for(dt), 96, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass2_d96(dt: DType) -> BenchSetup { - pass2(sdpa_decode_2pass_pass2_d96::kernel_ir_for(dt), 96, dt) + pass2(mt_sdpa_decode_2pass_pass2_d96::kernel_ir_for(dt), 96, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass1_d256(dt: DType) -> BenchSetup { - pass1(sdpa_decode_2pass_pass1_d256::kernel_ir_for(dt), 256, dt) + pass1(mt_sdpa_decode_2pass_pass1_d256::kernel_ir_for(dt), 256, dt) } #[bench(dtypes = [f32, f16, bf16])] fn bench_pass2_d256(dt: DType) -> BenchSetup { - pass2(sdpa_decode_2pass_pass2_d256::kernel_ir_for(dt), 256, dt) + pass2(mt_sdpa_decode_2pass_pass2_d256::kernel_ir_for(dt), 256, dt) } } diff --git a/crates/metaltile-std/src/ffai/sdpa_decode_batched.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_batched.rs similarity index 97% rename from crates/metaltile-std/src/ffai/sdpa_decode_batched.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_decode_batched.rs index c47806bd..0e29b6c2 100644 --- a/crates/metaltile-std/src/ffai/sdpa_decode_batched.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_batched.rs @@ -4,8 +4,8 @@ //! positions share one KV walk per dispatch, amortizing KV memory //! bandwidth by K× vs. K independent single-Q `sdpa_decode` dispatches. //! -//! This file ships the K=2 (`sdpa_decode_batched_q2`) and K=4 -//! (`sdpa_decode_batched_q4`) decode-form specializations. K=8 and +//! This file ships the K=2 (`mt_sdpa_decode_batched_q2`) and K=4 +//! (`mt_sdpa_decode_batched_q4`) decode-form specializations. K=8 and //! K=16 land separately via prefill-tile reuse through //! `mt_sdpa_prefill_mma` (the FA-2 tile already implements the //! KV-reuse pattern at BQ × BK; the batched variant arm dispatches it @@ -14,11 +14,11 @@ //! //! ## DISPATCH INVARIANTS //! -//! - **K=2 (`sdpa_decode_batched_q2`):** TPG = 1024 (32 simdgroups × 32 +//! - **K=2 (`mt_sdpa_decode_batched_q2`):** TPG = 1024 (32 simdgroups × 32 //! lanes). The kernel emits MSL that pins this layout via the //! threadgroup-memory allocations and the cross-simdgroup reduction; //! any other TPG breaks the bank-conflict-padded `tg_out` stride. -//! - **K=4 (`sdpa_decode_batched_q4`):** TPG ≤ 768; recommended 512 +//! - **K=4 (`mt_sdpa_decode_batched_q4`):** TPG ≤ 768; recommended 512 //! (16 simdgroups × 32 lanes). **Dispatching at 1024 silently //! produces all-zero outputs** — Metal's pipeline-state compiler caps //! `maxTotalThreadsPerThreadgroup` at 768 for this kernel's register @@ -88,7 +88,7 @@ use metaltile::kernel; #[kernel] -pub fn sdpa_decode_batched_q2( +pub fn mt_sdpa_decode_batched_q2( q: Tensor, k: Tensor, v: Tensor, @@ -358,7 +358,7 @@ pub fn sdpa_decode_batched_q2( // `sdpa_decode`'s sink + window passes. #[kernel] -pub fn sdpa_decode_batched_q4( +pub fn mt_sdpa_decode_batched_q4( q: Tensor, k: Tensor, v: Tensor, @@ -707,7 +707,7 @@ pub fn sdpa_decode_batched_q4( // Each phase's body duplicates the established pattern. #[kernel] -pub fn sdpa_decode_batched_q8( +pub fn mt_sdpa_decode_batched_q8( q: Tensor, k: Tensor, v: Tensor, @@ -1299,12 +1299,12 @@ mod tests { core::{DType, ir::KernelMode}, }; - use super::sdpa_decode_batched_q2; + use super::mt_sdpa_decode_batched_q2; fn msl_for(dt: DType) -> String { - let mut k = sdpa_decode_batched_q2::kernel_ir_for(dt); + let mut k = mt_sdpa_decode_batched_q2::kernel_ir_for(dt); k.mode = KernelMode::Reduction; - MslGenerator::default().generate(&k).expect("sdpa_decode_batched_q2 codegen succeeds") + MslGenerator::default().generate(&k).expect("mt_sdpa_decode_batched_q2 codegen succeeds") } #[test] @@ -1313,7 +1313,7 @@ mod tests { let src = msl_for(dt); assert!(!src.trim().is_empty(), "MSL for {dt:?} should not be empty"); assert!( - src.contains("kernel void sdpa_decode_batched_q2"), + src.contains("kernel void mt_sdpa_decode_batched_q2"), "MSL for {dt:?} should declare the kernel function:\n{src}", ); } @@ -1376,9 +1376,9 @@ mod tests { // ── K=4 codegen tests ──────────────────────────────────────────── fn msl_for_q4(dt: DType) -> String { - let mut k = super::sdpa_decode_batched_q4::kernel_ir_for(dt); + let mut k = super::mt_sdpa_decode_batched_q4::kernel_ir_for(dt); k.mode = KernelMode::Reduction; - MslGenerator::default().generate(&k).expect("sdpa_decode_batched_q4 codegen succeeds") + MslGenerator::default().generate(&k).expect("mt_sdpa_decode_batched_q4 codegen succeeds") } #[test] @@ -1387,7 +1387,7 @@ mod tests { let src = msl_for_q4(dt); assert!(!src.trim().is_empty(), "MSL for {dt:?} should not be empty"); assert!( - src.contains("kernel void sdpa_decode_batched_q4"), + src.contains("kernel void mt_sdpa_decode_batched_q4"), "MSL for {dt:?} should declare the kernel function:\n{src}", ); } @@ -1445,7 +1445,7 @@ mod tests { pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::{sdpa_decode_batched_q2, sdpa_decode_batched_q4, sdpa_decode_batched_q8}; + use super::{mt_sdpa_decode_batched_q2, mt_sdpa_decode_batched_q4, mt_sdpa_decode_batched_q8}; use crate::utils::{pack_f32, unpack_f32}; // Dense SDPA for a single Q slot: `[n_q_heads, head_dim]` query, @@ -1572,19 +1572,19 @@ pub mod kernel_tests { #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_sdpa_decode_batched_q2(dt: DType) -> TestSetup { - batched_setup(sdpa_decode_batched_q2::kernel_ir_for(dt), 2, 1024, dt) + batched_setup(mt_sdpa_decode_batched_q2::kernel_ir_for(dt), 2, 1024, dt) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_sdpa_decode_batched_q4(dt: DType) -> TestSetup { - batched_setup(sdpa_decode_batched_q4::kernel_ir_for(dt), 4, 512, dt) + batched_setup(mt_sdpa_decode_batched_q4::kernel_ir_for(dt), 4, 512, dt) } #[test_kernel(dtypes = [f32, f16, bf16], tol = [1e-3, 2e-3, 1e-2])] fn test_sdpa_decode_batched_q8(dt: DType) -> TestSetup { // q8 carries the most per-thread state — dispatch at 256 (8 simdgroups) // to stay within the Apple GPU register file (512 still miscomputes). - batched_setup(sdpa_decode_batched_q8::kernel_ir_for(dt), 8, 256, dt) + batched_setup(mt_sdpa_decode_batched_q8::kernel_ir_for(dt), 8, 256, dt) } } @@ -1594,7 +1594,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::{sdpa_decode_batched_q2, sdpa_decode_batched_q4, sdpa_decode_batched_q8}; + use super::{mt_sdpa_decode_batched_q2, mt_sdpa_decode_batched_q4, mt_sdpa_decode_batched_q8}; fn setup(ir: metaltile::core::ir::Kernel, batch_q: usize, dt: DType) -> BenchSetup { let (n_q_heads, n_kv_heads, head_dim) = (32usize, 8usize, 128usize); @@ -1621,11 +1621,17 @@ pub mod kernel_benches { } #[bench(dtypes = [f32, f16, bf16])] - fn bench_q2(dt: DType) -> BenchSetup { setup(sdpa_decode_batched_q2::kernel_ir_for(dt), 2, dt) } + fn bench_q2(dt: DType) -> BenchSetup { + setup(mt_sdpa_decode_batched_q2::kernel_ir_for(dt), 2, dt) + } #[bench(dtypes = [f32, f16, bf16])] - fn bench_q4(dt: DType) -> BenchSetup { setup(sdpa_decode_batched_q4::kernel_ir_for(dt), 4, dt) } + fn bench_q4(dt: DType) -> BenchSetup { + setup(mt_sdpa_decode_batched_q4::kernel_ir_for(dt), 4, dt) + } #[bench(dtypes = [f32, f16, bf16])] - fn bench_q8(dt: DType) -> BenchSetup { setup(sdpa_decode_batched_q8::kernel_ir_for(dt), 8, dt) } + fn bench_q8(dt: DType) -> BenchSetup { + setup(mt_sdpa_decode_batched_q8::kernel_ir_for(dt), 8, dt) + } } diff --git a/crates/metaltile-std/src/ffai/sdpa_decode_d512_sink.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_d512_sink.rs similarity index 96% rename from crates/metaltile-std/src/ffai/sdpa_decode_d512_sink.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_decode_d512_sink.rs index 544168a9..50e7e612 100644 --- a/crates/metaltile-std/src/ffai/sdpa_decode_d512_sink.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_d512_sink.rs @@ -15,7 +15,7 @@ //! layers — the model's `attn_sink` parameter is a `[n_heads]` fp32 //! tensor learned alongside Q/K/V/O. //! -//! Clone of [`crate::ffai::sdpa_decode_d512`] with the sink fold +//! Clone of [`crate::kernels::sdpa::sdpa_decode`] with the sink fold //! applied in the cross-simdgroup reduction (where `g_max` and `g_sum` //! are finalised). All other dispatch invariants — TPG=512, 16 dims //! per lane, 4-phase output reduction — are identical. @@ -23,7 +23,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_decode_d512_sink( +pub fn mt_sdpa_decode_d512_sink( q: Tensor, k: Tensor, v: Tensor, @@ -290,12 +290,12 @@ mod tests { core::{DType, ir::KernelMode}, }; - use super::ffai_sdpa_decode_d512_sink; + use super::mt_sdpa_decode_d512_sink; fn msl_for(dt: DType) -> String { - let mut k = ffai_sdpa_decode_d512_sink::kernel_ir_for(dt); + let mut k = mt_sdpa_decode_d512_sink::kernel_ir_for(dt); k.mode = KernelMode::Reduction; - MslGenerator::default().generate(&k).expect("ffai_sdpa_decode_d512_sink codegen succeeds") + MslGenerator::default().generate(&k).expect("mt_sdpa_decode_d512_sink codegen succeeds") } #[test] @@ -304,8 +304,8 @@ mod tests { let src = msl_for(dt); assert!(!src.trim().is_empty(), "MSL for {dt:?} should not be empty"); assert!( - src.contains("kernel void ffai_sdpa_decode_d512_sink"), - "MSL for {dt:?} should declare ffai_sdpa_decode_d512_sink:\n{src}", + src.contains("kernel void mt_sdpa_decode_d512_sink"), + "MSL for {dt:?} should declare mt_sdpa_decode_d512_sink:\n{src}", ); } } @@ -315,7 +315,7 @@ mod tests { pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_decode_d512_sink; + use super::mt_sdpa_decode_d512_sink; use crate::utils::{pack_f32, unpack_f32}; /// Dense softmax-attention oracle with a per-head `sink_logit` @@ -390,7 +390,7 @@ pub mod kernel_tests { let expected = naive_sdpa_sink( &q, &k, &v, &sink, n_q_heads, n_kv_heads, head_dim, n_kv, kv_stride, scale, ); - TestSetup::new(ffai_sdpa_decode_d512_sink::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_d512_sink::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -410,7 +410,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_decode_d512_sink; + use super::mt_sdpa_decode_d512_sink; #[bench(dtypes = [f32, f16, bf16])] fn bench_sdpa_decode_d512_sink(dt: DType) -> BenchSetup { @@ -420,7 +420,7 @@ pub mod kernel_benches { let scale = 1.0f32 / (head_dim as f32).sqrt(); let bytes = (2 * n_q_heads * head_dim + 2 * n_kv_heads * n_kv * head_dim) * dt.size_bytes() + n_q_heads * 4; - BenchSetup::new(ffai_sdpa_decode_d512_sink::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode_d512_sink::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_decode_sink_buf.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_sink_buf.rs similarity index 98% rename from crates/metaltile-std/src/ffai/sdpa_decode_sink_buf.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_decode_sink_buf.rs index 9f2d1096..48ea80fa 100644 --- a/crates/metaltile-std/src/ffai/sdpa_decode_sink_buf.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_decode_sink_buf.rs @@ -31,7 +31,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_decode_sink_buf( +pub fn mt_sdpa_decode_sink_buf( q: Tensor, k: Tensor, v: Tensor, @@ -207,7 +207,7 @@ pub fn ffai_sdpa_decode_sink_buf( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_decode_sink_buf; + use super::mt_sdpa_decode_sink_buf; use crate::utils::{pack_f32, unpack_f32}; /// Single-query decode oracle with a per-head sink (value 0) folded @@ -287,7 +287,7 @@ pub mod kernel_tests { let expected = naive(&q, &k, &v, &sink, n_q_heads, n_kv_heads, head_dim, n_kv, kv_stride, scale); - TestSetup::new(ffai_sdpa_decode_sink_buf::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_decode_sink_buf::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -322,7 +322,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_decode_sink_buf; + use super::mt_sdpa_decode_sink_buf; #[bench(dtypes = [f32, f16, bf16])] fn bench_sdpa_decode_sink_buf(dt: DType) -> BenchSetup { @@ -333,7 +333,7 @@ pub mod kernel_benches { let heads_per_group = n_q_heads / n_kv_heads; let scale = 1.0f32 / (head_dim as f32).sqrt(); let bytes = (n_q_heads * head_dim + 2 * n_kv_heads * n_kv * head_dim) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_decode_sink_buf::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_decode_sink_buf::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_multi.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_multi.rs similarity index 95% rename from crates/metaltile-std/src/ffai/sdpa_multi.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_multi.rs index f1196187..03b4316e 100644 --- a/crates/metaltile-std/src/ffai/sdpa_multi.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_multi.rs @@ -6,7 +6,7 @@ //! block of tokens is forwarded at once instead of one decode step at //! a time. //! -//! This is `ffai_sdpa_decode` generalised with a query dimension: one +//! This is `mt_sdpa_decode` generalised with a query dimension: one //! threadgroup per (query, q_head), the same TPG=1024 online-softmax //! cross-simdgroup reduction. Two attention modes select via the //! `causal` uniform: @@ -21,7 +21,7 @@ //! ## DISPATCH INVARIANTS //! //! Reduction-mode kernel — STRICT threadgroup geometry, the same -//! machine-freeze hazard as `ffai_sdpa_decode`. Consumers MUST encode +//! machine-freeze hazard as `mt_sdpa_decode`. Consumers MUST encode //! these as preconditions in their wrappers. //! //! - **TPG = 1024 threads** (32 simdgroups × 32 lanes). Hard. A TPG @@ -44,7 +44,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_multi( +pub fn mt_sdpa_multi( q: Tensor, k: Tensor, v: Tensor, @@ -92,7 +92,7 @@ pub fn ffai_sdpa_multi( // per-lane quartile dot product into the full score; online softmax // updates the running (max, sum); V accumulates into fp32 registers. // Pre-compute the kv VIDs before the loads so vectorize sees 4 - // consecutive Op::Load (same constraint as ffai_sdpa_decode). + // consecutive Op::Load (same constraint as mt_sdpa_decode). for _t in range(sg, n_kv, ns) { let base = kv_head_base + _t * head_dim; let kv_idx = base + d0; @@ -151,7 +151,7 @@ pub fn ffai_sdpa_multi( let g_sum = threadgroup_load("tg_sum", 0); let rescale = select(g_sum > 0.0f32, exp(run_max - g_max) / g_sum, 0.0f32); // Transpose-then-reduce with a +1 padded stride so adjacent lanes - // hit distinct threadgroup-memory banks (see ffai_sdpa_decode). + // hit distinct threadgroup-memory banks (see mt_sdpa_decode). let stride = ns + 1u32; let idx = lane * stride + sg; threadgroup_store("tg_out0", idx, o0 * rescale); @@ -179,9 +179,9 @@ pub fn ffai_sdpa_multi( } } -// ─── Tree-causal variant: `ffai_sdpa_multi_tree_mask` ─────────────── +// ─── Tree-causal variant: `mt_sdpa_multi_tree_mask` ─────────────── // -// Identical to `ffai_sdpa_multi` except the `causal: u32` constexpr is +// Identical to `mt_sdpa_multi` except the `causal: u32` constexpr is // replaced by a runtime additive `mask: Tensor` of shape // `[n_query, n_query]`. The mask is consulted ONLY for in-block KV // positions (i.e. `_t >= base_kv`); the cached prefix (`_t < base_kv`) @@ -198,10 +198,10 @@ pub fn ffai_sdpa_multi( // Equivalence: `treeCausalMask` from FFAI's `DraftTreeNode` is the // canonical mask producer. // -// Dispatch invariants — IDENTICAL to `ffai_sdpa_multi`. TPG=1024, +// Dispatch invariants — IDENTICAL to `mt_sdpa_multi`. TPG=1024, // head_dim=128 hard, grid `[n_q_heads * n_query * 1024, 1, 1]`. #[kernel] -pub fn ffai_sdpa_multi_tree_mask( +pub fn mt_sdpa_multi_tree_mask( q: Tensor, k: Tensor, v: Tensor, @@ -338,7 +338,7 @@ pub fn ffai_sdpa_multi_tree_mask( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_multi; + use super::mt_sdpa_multi; use crate::utils::{pack_f32, unpack_f32}; // Per (query, q_head): softmax(Q·Kᵀ·scale)·V over the attended KV @@ -414,7 +414,7 @@ pub mod kernel_tests { &q, &k, &v, n_q_heads, n_kv_heads, head_dim, base_kv, n_query, kv_stride, causal, scale, ); - TestSetup::new(ffai_sdpa_multi::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_multi::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -442,11 +442,11 @@ pub mod kernel_tests { fn test_ffai_sdpa_multi_causal(dt: DType) -> TestSetup { multi_setup(dt, true) } } -/// New-syntax benchmark for `ffai_sdpa_multi` (`class=GenericEmpty`). +/// New-syntax benchmark for `mt_sdpa_multi` (`class=GenericEmpty`). pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_multi; + use super::mt_sdpa_multi; #[bench(dtypes = [f32, f16, bf16])] fn bench_sdpa_multi(dt: DType) -> BenchSetup { @@ -458,7 +458,7 @@ pub mod kernel_benches { let n_kv = base_kv + n_query; let bytes = (2 * n_query * n_q_heads * head_dim + 2 * n_kv_heads * n_kv * head_dim) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_multi::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_multi::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) @@ -480,7 +480,7 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16])] fn bench_sdpa_multi_tree_mask(dt: DType) -> BenchSetup { - use super::ffai_sdpa_multi_tree_mask; + use super::mt_sdpa_multi_tree_mask; let (n_q_heads, n_kv_heads, head_dim) = (32usize, 8usize, 128usize); let (base_kv, n_query) = (4096usize, 8usize); let kv_stride = base_kv + n_query; @@ -491,7 +491,7 @@ pub mod kernel_benches { + 2 * n_kv_heads * n_kv * head_dim + n_query * n_query) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_multi_tree_mask::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_multi_tree_mask::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_multi_d256.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_multi_d256.rs similarity index 95% rename from crates/metaltile-std/src/ffai/sdpa_multi_d256.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_multi_d256.rs index 8ae3008f..67144239 100644 --- a/crates/metaltile-std/src/ffai/sdpa_multi_d256.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_multi_d256.rs @@ -1,5 +1,5 @@ //! Multi-query SDPA for `head_dim == 256`. The d=256 sibling of -//! `ffai_sdpa_multi`, generalising `ffai_sdpa_decode_d256` with a +//! `mt_sdpa_multi`, generalising `mt_sdpa_decode_d256` with a //! query dimension. One threadgroup per (query, q_head) attends a //! shared K/V cache in a single dispatch, TPG=1024 online softmax, //! same multi-query causal / full semantics as the d=128 reference @@ -22,8 +22,8 @@ //! ## DISPATCH INVARIANTS //! //! Reduction-mode kernel, STRICT threadgroup geometry, same -//! machine-freeze hazard as `ffai_sdpa_decode_d256` and -//! `ffai_sdpa_multi`. Consumers MUST encode these as preconditions +//! machine-freeze hazard as `mt_sdpa_decode_d256` and +//! `mt_sdpa_multi`. Consumers MUST encode these as preconditions //! in their wrappers. //! //! - **TPG = 1024 threads** (32 simdgroups × 32 lanes). Hard. A TPG @@ -53,12 +53,12 @@ //! over Apple's per-kernel threadgroup-memory cap. We split into two //! halves (dims 0..3 then 4..7) and reuse the same 4 tg_out buffers //! across both phases. Two extra barriers, same ~16 KB allocation. -//! This mirrors `ffai_sdpa_decode_d256` exactly. +//! This mirrors `mt_sdpa_decode_d256` exactly. use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_multi_d256( +pub fn mt_sdpa_multi_d256( q: Tensor, k: Tensor, v: Tensor, @@ -216,12 +216,12 @@ mod tests { core::{DType, ir::KernelMode}, }; - use super::ffai_sdpa_multi_d256; + use super::mt_sdpa_multi_d256; fn msl_for(dt: DType) -> String { - let mut k = ffai_sdpa_multi_d256::kernel_ir_for(dt); + let mut k = mt_sdpa_multi_d256::kernel_ir_for(dt); k.mode = KernelMode::Reduction; - MslGenerator::default().generate(&k).expect("ffai_sdpa_multi_d256 codegen succeeds") + MslGenerator::default().generate(&k).expect("mt_sdpa_multi_d256 codegen succeeds") } #[test] @@ -230,8 +230,8 @@ mod tests { let src = msl_for(dt); assert!(!src.trim().is_empty(), "MSL for {dt:?} should not be empty"); assert!( - src.contains("kernel void ffai_sdpa_multi_d256"), - "MSL for {dt:?} should declare ffai_sdpa_multi_d256:\n{src}", + src.contains("kernel void mt_sdpa_multi_d256"), + "MSL for {dt:?} should declare mt_sdpa_multi_d256:\n{src}", ); } } @@ -240,7 +240,7 @@ mod tests { pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_multi_d256; + use super::mt_sdpa_multi_d256; use crate::utils::{pack_f32, unpack_f32}; // Per (query, q_head): softmax(Q·Kᵀ·scale)·V over the attended KV @@ -318,7 +318,7 @@ pub mod kernel_tests { &q, &k, &v, n_q_heads, n_kv_heads, head_dim, base_kv, n_query, kv_stride, causal, scale, ); - TestSetup::new(ffai_sdpa_multi_d256::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_multi_d256::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -348,7 +348,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_multi_d256; + use super::mt_sdpa_multi_d256; #[bench(dtypes = [f32, f16, bf16])] fn bench_sdpa_multi_d256(dt: DType) -> BenchSetup { @@ -360,7 +360,7 @@ pub mod kernel_benches { let n_kv = base_kv + n_query; let bytes = (2 * n_query * n_q_heads * head_dim + 2 * n_kv_heads * n_kv * head_dim) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_multi_d256::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_multi_d256::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_prefill_d512_sink.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_prefill_d512_sink.rs similarity index 98% rename from crates/metaltile-std/src/ffai/sdpa_prefill_d512_sink.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_prefill_d512_sink.rs index c4b1821e..123b42b6 100644 --- a/crates/metaltile-std/src/ffai/sdpa_prefill_d512_sink.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_prefill_d512_sink.rs @@ -1,7 +1,7 @@ //! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric //! SPDX-License-Identifier: Apache-2.0 //! Multi-query causal sliding-window SDPA for head_dim=512 with attention -//! sink — the PREFILL counterpart of `ffai_sdpa_decode_d512_sink`. Each +//! sink — the PREFILL counterpart of `mt_sdpa_decode_d512_sink`. Each //! (q_head, q_pos) threadgroup runs the same flash online-softmax over the //! KV cache, but bounded causally: query at absolute position //! `p = kv_base + q_pos` attends KV `[max(0, p+1-window) .. p]`. For the @@ -15,7 +15,7 @@ use metaltile::kernel; #[kernel] #[allow(clippy::too_many_arguments)] -pub fn ffai_sdpa_prefill_d512_sink( +pub fn mt_sdpa_prefill_d512_sink( q: Tensor, k: Tensor, v: Tensor, @@ -280,7 +280,7 @@ pub fn ffai_sdpa_prefill_d512_sink( pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_prefill_d512_sink; + use super::mt_sdpa_prefill_d512_sink; #[bench(dtypes = [f32, f16, bf16])] fn bench_sdpa_prefill_d512_sink(dt: DType) -> BenchSetup { @@ -288,7 +288,7 @@ pub mod kernel_benches { let n_q = 64usize; let n_query = 256usize; let kv = 256usize; - BenchSetup::new(ffai_sdpa_prefill_d512_sink::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_prefill_d512_sink::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q * hd, dt)) .buffer(BenchBuffer::random("k", kv * hd, dt)) diff --git a/crates/metaltile-std/src/ffai/sdpa_rel_pos_conformer.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_rel_pos_conformer.rs similarity index 98% rename from crates/metaltile-std/src/ffai/sdpa_rel_pos_conformer.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_rel_pos_conformer.rs index 7b509c7e..a1840d2a 100644 --- a/crates/metaltile-std/src/ffai/sdpa_rel_pos_conformer.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_rel_pos_conformer.rs @@ -41,7 +41,7 @@ use metaltile::kernel; #[kernel] -pub fn ffai_sdpa_rel_pos_conformer_d128( +pub fn mt_sdpa_rel_pos_conformer_d128( q: Tensor, k: Tensor, v: Tensor, @@ -193,7 +193,7 @@ pub fn ffai_sdpa_rel_pos_conformer_d128( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::ffai_sdpa_rel_pos_conformer_d128; + use super::mt_sdpa_rel_pos_conformer_d128; use crate::utils::{pack_f32, unpack_f32}; // Per (query, q_head): softmax(scale·((q+u)·kᵀ + (q+v)·rel_embᵀ))·V @@ -297,7 +297,7 @@ pub mod kernel_tests { kv_stride, rel_zero, rel_len, scale, ); - TestSetup::new(ffai_sdpa_rel_pos_conformer_d128::kernel_ir_for(dt)) + TestSetup::new(mt_sdpa_rel_pos_conformer_d128::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("q", pack_f32(&q, dt), dt)) .input(TestBuffer::from_vec("k", pack_f32(&k, dt), dt)) @@ -332,7 +332,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::ffai_sdpa_rel_pos_conformer_d128; + use super::mt_sdpa_rel_pos_conformer_d128; #[bench(dtypes = [f32, f16, bf16])] fn bench_conformer(dt: DType) -> BenchSetup { @@ -347,7 +347,7 @@ pub mod kernel_benches { let n_kv = base_kv + n_query; let bytes = (2 * n_query * n_q_heads * head_dim + 2 * n_kv_heads * n_kv * head_dim) * dt.size_bytes(); - BenchSetup::new(ffai_sdpa_rel_pos_conformer_d128::kernel_ir_for(dt)) + BenchSetup::new(mt_sdpa_rel_pos_conformer_d128::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("q", n_query * n_q_heads * head_dim, dt)) .buffer(BenchBuffer::random("k", n_kv_heads * kv_stride * head_dim, dt)) diff --git a/crates/metaltile-std/src/mlx/sdpa_vector.rs b/crates/metaltile-std/src/kernels/sdpa/sdpa_vector.rs similarity index 99% rename from crates/metaltile-std/src/mlx/sdpa_vector.rs rename to crates/metaltile-std/src/kernels/sdpa/sdpa_vector.rs index c024acc5..c748e69b 100644 --- a/crates/metaltile-std/src/mlx/sdpa_vector.rs +++ b/crates/metaltile-std/src/kernels/sdpa/sdpa_vector.rs @@ -76,7 +76,7 @@ pub fn mt_sdpa_vector( // Pre-computing the 4 KV indices and issuing the 4 loads as a single run // (no BinOp/Cast interleaved) is what lets the vectorize pass collapse // them into one bfloat4 / half4 / float4 load — same shape as - // `sdpa_decode_2pass_pass1`. Inline'd loads + casts broke the run before. + // `mt_sdpa_decode_2pass_pass1`. Inline'd loads + casts broke the run before. for t in range(sg, n_kv, ns) { let kv0 = kv_base + t * head_dim + d0; let kv1 = kv0 + 1u32; @@ -363,7 +363,7 @@ pub fn mt_sdpa_vector_d96( /// Generic `T ∈ {f32, f16, bf16}`. TPG=1024; grid=[n_q_heads,1,1]. /// /// 6 live K accumulators + 6 V accumulators per lane. Register pressure is -/// higher than d=128 but below the d=256 bound that `ffai_sdpa_decode_d256` +/// higher than d=128 but below the d=256 bound that `mt_sdpa_decode_d256` /// confirmed fits in 1024 threads/TG (8 accumulators). #[kernel] pub fn mt_sdpa_vector_d192( @@ -492,7 +492,7 @@ pub fn mt_sdpa_vector_d192( /// GQA decode SDPA, head_dim=256. Each lane owns 8 elements (`256/32`). /// Generic `T ∈ {f32, f16, bf16}`. TPG=1024; grid=[n_q_heads,1,1]. /// -/// 8 live K accumulators + 8 V accumulators per lane. `ffai_sdpa_decode_d256` +/// 8 live K accumulators + 8 V accumulators per lane. `mt_sdpa_decode_d256` /// confirmed that 8-element/lane fits within the pipeline cap at 1024 /// threads/TG. Output reduction uses 8 sequential phases over the same /// 1024-slot `tg_out` buffer (same ~4 KB TG footprint as d=128). diff --git a/crates/metaltile-std/src/lib.rs b/crates/metaltile-std/src/lib.rs index df8ae016..3273429b 100644 --- a/crates/metaltile-std/src/lib.rs +++ b/crates/metaltile-std/src/lib.rs @@ -5,11 +5,9 @@ //! `metaltile-std` provides the kernel definitions (`#[kernel]` / `#[bench]` / //! `#[test_kernel]`) and the shared bench-setup utilities used by the kernels. -pub mod ffai; pub mod kernels; pub mod mlx; pub mod probe; -pub mod quant; // Re-export the kernel inventories from the harness registry. The `#[kernel]` / // `#[bench]` / `#[test_kernel]` registrations live in this crate's `ffai` / diff --git a/crates/metaltile-std/src/mlx/mod.rs b/crates/metaltile-std/src/mlx/mod.rs index 8047f0ee..04e8c58a 100644 --- a/crates/metaltile-std/src/mlx/mod.rs +++ b/crates/metaltile-std/src/mlx/mod.rs @@ -16,9 +16,6 @@ // block_scaled_dequant → quant family (migrated later); block_scaled_moe → // kernels/moe/. The quantized matmuls moved to kernels/gemm/. -pub mod block_scaled_dequant; -pub mod scaled_dot_product_attention; -pub mod sdpa_vector; pub mod steel; // `conv.rs` and `shared.rs` are placeholder/stale stubs left over from diff --git a/crates/metaltile-std/tests/aura_msl_snapshots.rs b/crates/metaltile-std/tests/aura_msl_snapshots.rs index f605c48c..57bf7980 100644 --- a/crates/metaltile-std/tests/aura_msl_snapshots.rs +++ b/crates/metaltile-std/tests/aura_msl_snapshots.rs @@ -32,13 +32,14 @@ use metaltile::{ codegen::{MslGenerator, msl::MslConfig}, core::{dtype::DType, ir::KernelMode}, }; -use metaltile_std::ffai::{ - aura_dequant_rotated::aura_dequant_rotated_int4, - aura_encode::aura_encode_int4, - aura_flash_p1::aura_flash_p1_kb4_vb2_d128, - aura_flash_pass2::aura_flash_pass2_d128, - aura_score::aura_score_int4, - aura_value::aura_value_int4, +use metaltile_std::kernels::{ + quant::{aura_dequant_rotated::mt_aura_dequant_rotated_int4, aura_encode::mt_aura_encode_int4}, + sdpa::{ + aura_flash_p1::mt_aura_flash_p1_kb4_vb2_d128, + aura_flash_pass2::mt_aura_flash_pass2_d128, + aura_score::mt_aura_score_int4, + aura_value::mt_aura_value_int4, + }, }; /// Generate MSL for an AURA kernel's IR with its declared kernel mode. @@ -53,50 +54,51 @@ fn aura_msl(kernel_ir: metaltile::core::ir::Kernel, mode: KernelMode) -> String .expect("AURA kernel must codegen cleanly") } -/// `aura_encode` — fused L2-norm + rotation + Lloyd-Max quantize + +/// `mt_aura_encode` — fused L2-norm + rotation + Lloyd-Max quantize + /// bit-pack. Reduction mode (`simd_sum` over the rotated coordinates). #[test] -fn aura_encode_int4_f32_msl() { - let msl = aura_msl(aura_encode_int4::kernel_ir_for(DType::F32), KernelMode::Reduction); +fn mt_aura_encode_int4_f32_msl() { + let msl = aura_msl(mt_aura_encode_int4::kernel_ir_for(DType::F32), KernelMode::Reduction); assert_snapshot!(msl); } -/// `aura_dequant_rotated` — bulk unpack + de-rotate of a packed AURA +/// `mt_aura_dequant_rotated` — bulk unpack + de-rotate of a packed AURA /// K/V slab. Grid3D mode (one thread per packed word). #[test] -fn aura_dequant_rotated_int4_f32_msl() { - let msl = aura_msl(aura_dequant_rotated_int4::kernel_ir_for(DType::F32), KernelMode::Grid3D); +fn mt_aura_dequant_rotated_int4_f32_msl() { + let msl = aura_msl(mt_aura_dequant_rotated_int4::kernel_ir_for(DType::F32), KernelMode::Grid3D); assert_snapshot!(msl); } -/// `aura_score` — per-token Q·K score against the packed AURA cache. +/// `mt_aura_score` — per-token Q·K score against the packed AURA cache. /// Reduction mode. #[test] -fn aura_score_int4_f32_msl() { - let msl = aura_msl(aura_score_int4::kernel_ir_for(DType::F32), KernelMode::Reduction); +fn mt_aura_score_int4_f32_msl() { + let msl = aura_msl(mt_aura_score_int4::kernel_ir_for(DType::F32), KernelMode::Reduction); assert_snapshot!(msl); } -/// `aura_value` — softmax-weighted accumulation of the packed V cache. +/// `mt_aura_value` — softmax-weighted accumulation of the packed V cache. /// Grid3D mode. #[test] -fn aura_value_int4_f32_msl() { - let msl = aura_msl(aura_value_int4::kernel_ir_for(DType::F32), KernelMode::Grid3D); +fn mt_aura_value_int4_f32_msl() { + let msl = aura_msl(mt_aura_value_int4::kernel_ir_for(DType::F32), KernelMode::Grid3D); assert_snapshot!(msl); } -/// `aura_flash_p1` — flash-attention pass 1 over the packed cache +/// `mt_aura_flash_p1` — flash-attention pass 1 over the packed cache /// (kb4 / vb2 / d128 recipe). Grid3D mode. #[test] -fn aura_flash_p1_kb4_vb2_d128_f32_msl() { - let msl = aura_msl(aura_flash_p1_kb4_vb2_d128::kernel_ir_for(DType::F32), KernelMode::Grid3D); +fn mt_aura_flash_p1_kb4_vb2_d128_f32_msl() { + let msl = + aura_msl(mt_aura_flash_p1_kb4_vb2_d128::kernel_ir_for(DType::F32), KernelMode::Grid3D); assert_snapshot!(msl); } -/// `aura_flash_pass2` — flash-attention pass 2 reduction (d128 recipe). +/// `mt_aura_flash_pass2` — flash-attention pass 2 reduction (d128 recipe). /// Reduction mode; storage in bf16, online softmax in fp32. #[test] -fn aura_flash_pass2_d128_bf16_msl() { - let msl = aura_msl(aura_flash_pass2_d128::kernel_ir_for(DType::BF16), KernelMode::Reduction); +fn mt_aura_flash_pass2_d128_bf16_msl() { + let msl = aura_msl(mt_aura_flash_pass2_d128::kernel_ir_for(DType::BF16), KernelMode::Reduction); assert_snapshot!(msl); } diff --git a/crates/metaltile-std/tests/moe_bgemm_iq2xxs_mpp_correctness.rs b/crates/metaltile-std/tests/moe_bgemm_iq2xxs_mpp_correctness.rs index ed09e370..e10aee51 100644 --- a/crates/metaltile-std/tests/moe_bgemm_iq2xxs_mpp_correctness.rs +++ b/crates/metaltile-std/tests/moe_bgemm_iq2xxs_mpp_correctness.rs @@ -2,7 +2,7 @@ //! SPDX-License-Identifier: Apache-2.0 //! GPU correctness for `ffai::mt_moe_gather_bgemm_iq2xxs_mpp` — the //! prefill IQ2_XXS grouped BGEMM. Oracle: per-row IQ2_XXS dequant gemv -//! (same formula as ffai_gguf_dequant_iq2_xxs). Cosine ≥ 0.99 (MMA +//! (same formula as mt_gguf_dequant_iq2_xxs). Cosine ≥ 0.99 (MMA //! accumulation order differs from the scalar oracle). #![cfg(target_os = "macos")] diff --git a/crates/metaltile-std/tests/moe_bgemm_q2k_mpp_correctness.rs b/crates/metaltile-std/tests/moe_bgemm_q2k_mpp_correctness.rs index abf0a822..71f9b0c3 100644 --- a/crates/metaltile-std/tests/moe_bgemm_q2k_mpp_correctness.rs +++ b/crates/metaltile-std/tests/moe_bgemm_q2k_mpp_correctness.rs @@ -12,8 +12,8 @@ use common::{Dt, gpu_lock, pack_bytes, pack_u32_bytes, unpack_bytes}; use metaltile::{Context, core::ir::KernelMode}; use metaltile_std::kernels::moe::moe_bgemm_q2k_mpp::mt_moe_gather_bgemm_q2k_mpp; // Shared Q2_K output-index → (qs byte, 2-bit shift) map (see PR #264/#265): the -// kernel, quantizer, and this oracle all read the one definition in quant::gguf. -use metaltile_std::quant::gguf::q2_k_qpos; +// kernel, quantizer, and this oracle all read the one definition in kernels::quant::gguf. +use metaltile_std::kernels::quant::gguf::q2_k_qpos; fn xorshift(s: &mut u32) -> u32 { let mut x = *s; diff --git a/crates/metaltile-std/tests/moe_gather_down_q2k_correctness.rs b/crates/metaltile-std/tests/moe_gather_down_q2k_correctness.rs index 607a85a9..31d879b0 100644 --- a/crates/metaltile-std/tests/moe_gather_down_q2k_correctness.rs +++ b/crates/metaltile-std/tests/moe_gather_down_q2k_correctness.rs @@ -17,9 +17,9 @@ use common::{Dt, gpu_lock, pack_bytes, pack_u32_bytes, unpack_bytes}; use metaltile::{Context, core::ir::KernelMode}; use metaltile_std::kernels::moe::moe_gather_down_q2k::mt_moe_gather_down_q2k; // The Q2_K output-index → (qs byte, 2-bit shift) map is the single shared -// definition in `quant::gguf`: the kernel, the quantizer, and this oracle all +// definition in `kernels::quant::gguf`: the kernel, the quantizer, and this oracle all // read it, so the layout can't drift apart (getting it wrong was PR #264). -use metaltile_std::quant::gguf::q2_k_qpos; +use metaltile_std::kernels::quant::gguf::q2_k_qpos; const N_SLOTS: usize = 6; diff --git a/crates/metaltile-std/tests/moe_gather_gemv_iq2xxs_correctness.rs b/crates/metaltile-std/tests/moe_gather_gemv_iq2xxs_correctness.rs index 5f0441c1..9d609a51 100644 --- a/crates/metaltile-std/tests/moe_gather_gemv_iq2xxs_correctness.rs +++ b/crates/metaltile-std/tests/moe_gather_gemv_iq2xxs_correctness.rs @@ -39,7 +39,7 @@ fn frand(state: &mut u32) -> f32 { /// CPU reference: dequant the IQ2_XXS split buffers and dot each /// expert/row against `x`. Mirrors -/// `ffai_gguf_dequant_iq2_xxs` exactly. +/// `mt_gguf_dequant_iq2_xxs` exactly. fn reference( x: &[f32], qs_all: &[u32], diff --git a/crates/metaltile-std/tests/sdpa_decode_2pass_gpu.rs b/crates/metaltile-std/tests/sdpa_decode_2pass_gpu.rs index 06b1d920..51e497de 100644 --- a/crates/metaltile-std/tests/sdpa_decode_2pass_gpu.rs +++ b/crates/metaltile-std/tests/sdpa_decode_2pass_gpu.rs @@ -24,7 +24,10 @@ use std::{ use common::{Dt, SdpaShape, max_abs_diff, naive_sdpa_f32, pack_bytes, ramp, unpack_bytes}; use metaltile::{Context, DispatchSpec, ResidentBuffer, core::ir::KernelMode}; -use metaltile_std::ffai::sdpa_decode_2pass::{sdpa_decode_2pass_pass1, sdpa_decode_2pass_pass2}; +use metaltile_std::kernels::sdpa::sdpa_decode_2pass::{ + mt_sdpa_decode_2pass_pass1, + mt_sdpa_decode_2pass_pass2, +}; /// Serialise GPU dispatches across tests in this file. Cargo runs `#[test]` /// functions concurrently by default; under `cargo llvm-cov` the @@ -75,7 +78,7 @@ fn run_2pass( let partial_o_len = a.n_q_heads * a.blocks * a.head_dim; let partial_ml_len = a.n_q_heads * a.blocks; - let mut p1 = sdpa_decode_2pass_pass1::kernel_ir_for(dtype); + let mut p1 = mt_sdpa_decode_2pass_pass1::kernel_ir_for(dtype); p1.mode = KernelMode::Reduction; let mut p1_bufs: BTreeMap> = BTreeMap::new(); p1_bufs.insert("q".into(), pack_bytes(a.q, dt)); @@ -94,7 +97,7 @@ fn run_2pass( p1_bufs.insert("blocks".into(), (a.blocks as u32).to_le_bytes().to_vec()); p1_bufs.insert("scale".into(), a.scale.to_le_bytes().to_vec()); - let mut p2 = sdpa_decode_2pass_pass2::kernel_ir_for(dtype); + let mut p2 = mt_sdpa_decode_2pass_pass2::kernel_ir_for(dtype); p2.mode = KernelMode::Reduction; let mut p2_bufs: BTreeMap> = BTreeMap::new(); p2_bufs.insert("out".into(), vec![0u8; a.n_q_heads * a.head_dim * dt.bytes()]); @@ -487,13 +490,13 @@ fn sdpa_decode_2pass_capture() { // ── Additional head_dim correctness tests: d={64,96,256} ───────────────────── -use metaltile_std::ffai::sdpa_decode_2pass::{ - sdpa_decode_2pass_pass1_d64, - sdpa_decode_2pass_pass1_d96, - sdpa_decode_2pass_pass1_d256, - sdpa_decode_2pass_pass2_d64, - sdpa_decode_2pass_pass2_d96, - sdpa_decode_2pass_pass2_d256, +use metaltile_std::kernels::sdpa::sdpa_decode_2pass::{ + mt_sdpa_decode_2pass_pass1_d64, + mt_sdpa_decode_2pass_pass1_d96, + mt_sdpa_decode_2pass_pass1_d256, + mt_sdpa_decode_2pass_pass2_d64, + mt_sdpa_decode_2pass_pass2_d96, + mt_sdpa_decode_2pass_pass2_d256, }; /// Dispatch the d-specific pass1+pass2 pair and compare to the naive @@ -522,16 +525,16 @@ fn run_2pass_generic( // Pick the right pass1/pass2 kernel pair by head_dim. let (mut p1_kernel, mut p2_kernel) = match head_dim { 64 => ( - sdpa_decode_2pass_pass1_d64::kernel_ir_for(dtype), - sdpa_decode_2pass_pass2_d64::kernel_ir_for(dtype), + mt_sdpa_decode_2pass_pass1_d64::kernel_ir_for(dtype), + mt_sdpa_decode_2pass_pass2_d64::kernel_ir_for(dtype), ), 96 => ( - sdpa_decode_2pass_pass1_d96::kernel_ir_for(dtype), - sdpa_decode_2pass_pass2_d96::kernel_ir_for(dtype), + mt_sdpa_decode_2pass_pass1_d96::kernel_ir_for(dtype), + mt_sdpa_decode_2pass_pass2_d96::kernel_ir_for(dtype), ), 256 => ( - sdpa_decode_2pass_pass1_d256::kernel_ir_for(dtype), - sdpa_decode_2pass_pass2_d256::kernel_ir_for(dtype), + mt_sdpa_decode_2pass_pass1_d256::kernel_ir_for(dtype), + mt_sdpa_decode_2pass_pass2_d256::kernel_ir_for(dtype), ), _ => panic!("unsupported head_dim {head_dim} in run_2pass_generic"), }; diff --git a/crates/metaltile-std/tests/sdpa_decode_d512_sink_nkv1.rs b/crates/metaltile-std/tests/sdpa_decode_d512_sink_nkv1.rs index 760b12c5..ef2577d7 100644 --- a/crates/metaltile-std/tests/sdpa_decode_d512_sink_nkv1.rs +++ b/crates/metaltile-std/tests/sdpa_decode_d512_sink_nkv1.rs @@ -1,6 +1,6 @@ //! Copyright 2026 TheTom //! SPDX-License-Identifier: Apache-2.0 -//! Isolated check: ffai_sdpa_decode_d512_sink with n_kv=1 (single visible +//! Isolated check: mt_sdpa_decode_d512_sink with n_kv=1 (single visible //! KV — the first decode token), many q heads (MQA), per-head sink. The //! in-source test only covers n_kv=64; DSv4 token 0 hits n_kv=1. #![cfg(target_os = "macos")] @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use common::{Dt, gpu_lock, pack_bytes, unpack_bytes}; use metaltile::{Context, core::ir::KernelMode}; -use metaltile_std::ffai::sdpa_decode_d512_sink::ffai_sdpa_decode_d512_sink; +use metaltile_std::kernels::sdpa::sdpa_decode_d512_sink::mt_sdpa_decode_d512_sink; fn xorshift(s: &mut u32) -> u32 { let mut x = *s; @@ -70,7 +70,7 @@ fn sdpa_decode_d512_sink_nkv1_64heads() { b.insert("scale".into(), scale.to_le_bytes().to_vec()); let ctx = Context::new().unwrap(); - let mut kern = ffai_sdpa_decode_d512_sink::kernel_ir_for(dt.to_dtype()); + let mut kern = mt_sdpa_decode_d512_sink::kernel_ir_for(dt.to_dtype()); kern.mode = KernelMode::Reduction; let r = ctx.dispatch_with_grid(&kern, &b, &BTreeMap::new(), [n_q, 1, 1], [512, 1, 1]).unwrap(); let got = unpack_bytes(r.outputs.get("out").unwrap(), dt); diff --git a/crates/metaltile-std/tests/sdpa_decode_swa_gpu.rs b/crates/metaltile-std/tests/sdpa_decode_swa_gpu.rs index c0b904f2..dd018066 100644 --- a/crates/metaltile-std/tests/sdpa_decode_swa_gpu.rs +++ b/crates/metaltile-std/tests/sdpa_decode_swa_gpu.rs @@ -1,6 +1,6 @@ //! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric //! SPDX-License-Identifier: Apache-2.0 -//! Sliding-window + sink-token perf bench for `ffai::sdpa_decode`. +//! Sliding-window + sink-token perf bench for `kernels::sdpa::sdpa_decode`. //! //! Companion to the in-source `#[test_kernel]` correctness coverage in //! `src/ffai/sdpa_decode.rs` (`test_ffai_sdpa_decode` / @@ -39,7 +39,7 @@ use std::collections::BTreeMap; use common::{Dt, pack_bytes, ramp}; use metaltile::{Context, DispatchSpec, ResidentBuffer, core::ir::KernelMode}; -use metaltile_std::ffai::sdpa_decode::ffai_sdpa_decode as sdpa_decode; +use metaltile_std::kernels::sdpa::sdpa_decode::mt_sdpa_decode as sdpa_decode; // (n_q_heads, n_kv_heads, n_kv, window_size, sink_tokens). Qwen3-class // GQA shape (32 Q heads / 8 KV heads) at the long-context regimes where diff --git a/crates/metaltile-std/tests/sdpa_prefill_d512_sink_correctness.rs b/crates/metaltile-std/tests/sdpa_prefill_d512_sink_correctness.rs index 298df8d5..886d88a0 100644 --- a/crates/metaltile-std/tests/sdpa_prefill_d512_sink_correctness.rs +++ b/crates/metaltile-std/tests/sdpa_prefill_d512_sink_correctness.rs @@ -1,6 +1,6 @@ //! Copyright 2026 TheTom //! SPDX-License-Identifier: Apache-2.0 -//! GPU correctness for `ffai::ffai_sdpa_prefill_d512_sink` — multi-query +//! GPU correctness for `ffai::mt_sdpa_prefill_d512_sink` — multi-query //! causal sliding-window SDPA (d512, attn sink, MQA). Oracle: per-(q_pos, //! q_head) causal softmax with sink over the KV window. #![cfg(target_os = "macos")] @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use common::{Dt, gpu_lock, pack_bytes, unpack_bytes}; use metaltile::{Context, core::ir::KernelMode}; -use metaltile_std::ffai::sdpa_prefill_d512_sink::ffai_sdpa_prefill_d512_sink; +use metaltile_std::kernels::sdpa::sdpa_prefill_d512_sink::mt_sdpa_prefill_d512_sink; fn xorshift(s: &mut u32) -> u32 { let mut x = *s; @@ -90,7 +90,7 @@ fn run_case_nq(dt: Dt, n_q: usize, n_query: usize, kv: usize, window: usize, tol b.insert("scale".into(), scale.to_le_bytes().to_vec()); let ctx = Context::new().unwrap(); - let mut kern = ffai_sdpa_prefill_d512_sink::kernel_ir_for(dt.to_dtype()); + let mut kern = mt_sdpa_prefill_d512_sink::kernel_ir_for(dt.to_dtype()); kern.mode = KernelMode::Reduction; let r = ctx.dispatch_with_grid(&kern, &b, &BTreeMap::new(), [n_q, n_query, 1], [32, 1, 1]).unwrap(); diff --git a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_dequant_rotated_int4_f32_msl.snap b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_dequant_rotated_int4_f32_msl.snap similarity index 98% rename from crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_dequant_rotated_int4_f32_msl.snap rename to crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_dequant_rotated_int4_f32_msl.snap index f2c45bdf..42fff13a 100644 --- a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_dequant_rotated_int4_f32_msl.snap +++ b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_dequant_rotated_int4_f32_msl.snap @@ -1,13 +1,12 @@ --- source: crates/metaltile-std/tests/aura_msl_snapshots.rs -assertion_line: 67 expression: msl --- // Generated by MetalTile #include using namespace metal; -kernel void aura_dequant_rotated_int4( +kernel void mt_aura_dequant_rotated_int4( const device uint *packed [[buffer(0)]], const device float *norms [[buffer(1)]], const device float *codebook [[buffer(2)]], diff --git a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_encode_int4_f32_msl.snap b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_encode_int4_f32_msl.snap similarity index 99% rename from crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_encode_int4_f32_msl.snap rename to crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_encode_int4_f32_msl.snap index 9b5853a6..4ba6a19a 100644 --- a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_encode_int4_f32_msl.snap +++ b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_encode_int4_f32_msl.snap @@ -1,13 +1,12 @@ --- source: crates/metaltile-std/tests/aura_msl_snapshots.rs -assertion_line: 59 expression: msl --- // Generated by MetalTile #include using namespace metal; -kernel void aura_encode_int4( +kernel void mt_aura_encode_int4( const device float *input [[buffer(0)]], const device float *rotation [[buffer(1)]], const device float *boundaries [[buffer(2)]], diff --git a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_flash_p1_kb4_vb2_d128_f32_msl.snap b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_flash_p1_kb4_vb2_d128_f32_msl.snap similarity index 99% rename from crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_flash_p1_kb4_vb2_d128_f32_msl.snap rename to crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_flash_p1_kb4_vb2_d128_f32_msl.snap index 1bf3b4c8..dc20cd32 100644 --- a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_flash_p1_kb4_vb2_d128_f32_msl.snap +++ b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_flash_p1_kb4_vb2_d128_f32_msl.snap @@ -1,13 +1,12 @@ --- source: crates/metaltile-std/tests/aura_msl_snapshots.rs -assertion_line: 91 expression: msl --- // Generated by MetalTile #include using namespace metal; -kernel void aura_flash_p1_kb4_vb2_d128( +kernel void mt_aura_flash_p1_kb4_vb2_d128( const device float *q_rot [[buffer(0)]], const device uint *key_packed [[buffer(1)]], const device float *key_norms [[buffer(2)]], diff --git a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_flash_pass2_d128_bf16_msl.snap b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_flash_pass2_d128_bf16_msl.snap similarity index 99% rename from crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_flash_pass2_d128_bf16_msl.snap rename to crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_flash_pass2_d128_bf16_msl.snap index 0c8947ee..6d515c20 100644 --- a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_flash_pass2_d128_bf16_msl.snap +++ b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_flash_pass2_d128_bf16_msl.snap @@ -6,7 +6,7 @@ expression: msl #include using namespace metal; -kernel void aura_flash_pass2_d128( +kernel void mt_aura_flash_pass2_d128( const device bfloat *o_partials [[buffer(0)]], const device bfloat *m_partials [[buffer(1)]], const device bfloat *l_partials [[buffer(2)]], diff --git a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_score_int4_f32_msl.snap b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_score_int4_f32_msl.snap similarity index 99% rename from crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_score_int4_f32_msl.snap rename to crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_score_int4_f32_msl.snap index d3ea73aa..a9eed6db 100644 --- a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_score_int4_f32_msl.snap +++ b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_score_int4_f32_msl.snap @@ -6,7 +6,7 @@ expression: msl #include using namespace metal; -kernel void aura_score_int4( +kernel void mt_aura_score_int4( const device float *q_rot [[buffer(0)]], const device uint *packed [[buffer(1)]], const device float *norms [[buffer(2)]], diff --git a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_value_int4_f32_msl.snap b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_value_int4_f32_msl.snap similarity index 98% rename from crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_value_int4_f32_msl.snap rename to crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_value_int4_f32_msl.snap index 7eef9b7b..64e79af6 100644 --- a/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__aura_value_int4_f32_msl.snap +++ b/crates/metaltile-std/tests/snapshots/aura_msl_snapshots__mt_aura_value_int4_f32_msl.snap @@ -6,7 +6,7 @@ expression: msl #include using namespace metal; -kernel void aura_value_int4( +kernel void mt_aura_value_int4( const device float *weights [[buffer(0)]], const device uint *packed [[buffer(1)]], const device float *norms [[buffer(2)]], diff --git a/crates/metaltile-std/tests/vulkan_sdpa_multi.rs b/crates/metaltile-std/tests/vulkan_sdpa_multi.rs index 1a82e304..7cfeea31 100644 --- a/crates/metaltile-std/tests/vulkan_sdpa_multi.rs +++ b/crates/metaltile-std/tests/vulkan_sdpa_multi.rs @@ -3,7 +3,7 @@ //! Copyright 2026 TheTom //! SPDX-License-Identifier: Apache-2.0 //! Vulkan/RDNA4 correctness for the multi-position flash-attention kernel -//! `ffai_sdpa_multi` (the prefill attention kernel). The existing +//! `mt_sdpa_multi` (the prefill attention kernel). The existing //! `#[test_kernel]` corpus exercises a small (n_query=8, kv_stride=64, //! GQA 8/4) shape; this test pins the *prefill-realistic* shapes the //! FFAI consumer dispatches — large `kv_stride` (cache capacity), a real @@ -25,7 +25,7 @@ use std::collections::BTreeMap; use metaltile::{VulkanDevice, core::dtype::DType}; -use metaltile_std::ffai::sdpa_multi::ffai_sdpa_multi; +use metaltile_std::kernels::sdpa::sdpa_multi::mt_sdpa_multi; fn xorshift(s: &mut u32) -> u32 { let mut x = *s; @@ -125,7 +125,7 @@ fn run_case( ); let dt = DType::F32; - let mut kernel = ffai_sdpa_multi::kernel_ir_for(dt); + let mut kernel = mt_sdpa_multi::kernel_ir_for(dt); kernel.mode = metaltile::core::ir::KernelMode::Reduction; let mut buffers: BTreeMap> = BTreeMap::new(); diff --git a/docs/specs/KERNEL_CONSOLIDATION_PLAN.md b/docs/specs/KERNEL_CONSOLIDATION_PLAN.md index a67106e8..761e65f7 100644 --- a/docs/specs/KERNEL_CONSOLIDATION_PLAN.md +++ b/docs/specs/KERNEL_CONSOLIDATION_PLAN.md @@ -52,7 +52,7 @@ crates/metaltile-std/src/kernels/ · dequant_gemv · gemm_q8(_mpp)/q4_mpp · batched_{qkv,4}(_block_scaled)_{qgemv,qmm} · patch_embed(_mma)_block_scaled · gemv_quantized (Q8/Q4 inline-dequant gemv, ex-gemv_q8 grab-bag) (same folder; format-axis fold deferred, §7) - sdpa/ ALL attention: bidirectional(+relpos/windowed/conformer) · decode(+d64..d512/ + sdpa/ ✅ DONE — ALL attention: bidirectional(+relpos/windowed/conformer) · decode(+d64..d512/ 2pass/batched/sink) · multi(+d256/tree-mask) · prefill_mma · flash_quantized · aura_flash · steel/attn moe/ ✅ DONE — router_topk · permute (+unpermute) · gather_qmm (per-expert BGEMM) · @@ -67,7 +67,7 @@ crates/metaltile-std/src/kernels/ convolution/ ✅ DONE — conv1d/2d/3d · depthwise · winograd · steel_conv · conv1d_causal(_roll) (see §4) ssm/ ✅ DONE — ssm(_replay) · gated_delta(+wy/prep/chunk) · mamba pregate-rmsnorm (gated_group_rmsnorm(_batched)) · softplus_add(_rows) - quant/ INFRA + the op×format matrix (§7): codec · format · gguf · block_scaled_* · + quant/ ✅ DONE (kernels/quant/) — INFRA + the op×format matrix (§7): codec · format · gguf · block_scaled_* · quantized_* · fp_quantized_* · affine · aura codec stack · dequant_* audio/ ✅ DONE — mel_spectrogram(+magnitude/stft/filterbank) · lstm · vocoder · snake1d · upsample vision/ ✅ DONE — resize_normalize(+bicubic) · im2col · patch_unfold · pos_emb_2d · avg_pool2d · @@ -170,7 +170,7 @@ payoff last: |---|---|---|---| | ✅ done | `convolution/`, `rope/`, `norm/`, `sampling/`, `ops/` | exemplar + all of wave 1 | 24k → ~1.6k | | 2 | ✅ `audio/` `vision/` `kv_cache/` `gemm/` (dense + quantized) `ssm/` all done | moderate size, few cross-deps | medium | -| 3 | ✅ `moe/` done; remaining `sdpa/`, **`quant/`** | hardest axes (head-dim d64..d512; bm8/bm64×int8; the 30-format matrix) — most of the ~150k LOC | the bulk | +| 3 | ✅ `moe/` `sdpa/` `quant/` all done; `kernels/quant/` unifies the host codec/format/gguf SSOT + the dequant kernels. `ffai/` is fully migrated and removed; only `mlx/steel/` (steel attn/conv templates) remains. | hardest axes (head-dim d64..d512; bm8/bm64×int8; the 30-format matrix) — most of the ~150k LOC | the bulk | ## 7. The `quant/` umbrella — collapsing the op × format matrix