diff --git a/crates/metaltile-codegen/src/spirv/mod.rs b/crates/metaltile-codegen/src/spirv/mod.rs index fc198bc0..a3a2ea3a 100644 --- a/crates/metaltile-codegen/src/spirv/mod.rs +++ b/crates/metaltile-codegen/src/spirv/mod.rs @@ -1057,7 +1057,7 @@ impl GlslGenerator { // step on top of the hardware reciprocal gives correctly- // rounded f32 divide assuming hardware fma is correctly // rounded (true on AMD GFX1xxx V_FMAC_F32). Without this - // `test_logits_repetition_penalty` misses bit-exact tol=0 + // `test_mt_logits_repetition_penalty` misses bit-exact tol=0 // by 1 ULP on 2 elements out of 8192. if matches!(bop, BinOpKind::Div) && ty == "float" { writeln!(out, "{pad}{ty} {v} = mt_fdiv({l}, {r});").ok(); diff --git a/crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs b/crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs index ace8f801..838aa33b 100644 --- a/crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs +++ b/crates/metaltile-std/src/ffai/dsv4_indexer_topk.rs @@ -14,7 +14,7 @@ //! shared memory. We do a parallel bitonic sort **descending** over //! `(score, original_index)` pairs and emit the first `k` indices. //! Same Batcher-pattern compare-and-swap as -//! [`crate::mlx::sort::mt_sort`] but with parallel storage for an +//! [`crate::kernels::sampling::sort::mt_sort`] but with parallel storage for an //! `u32` index tag so the original cache-position survives the swaps. //! //! Scores below `n_kv` are padded with `-INFINITY` so the sort diff --git a/crates/metaltile-std/src/ffai/mod.rs b/crates/metaltile-std/src/ffai/mod.rs index 983ebd9a..c54b3dbd 100644 --- a/crates/metaltile-std/src/ffai/mod.rs +++ b/crates/metaltile-std/src/ffai/mod.rs @@ -79,10 +79,6 @@ pub mod im2col_patch_interleaved; pub mod kv_cache; pub mod kv_cache_update_many; pub mod leaky_relu; -pub mod logits_min_p; -pub mod logits_processors; -pub mod logits_top_p; -pub mod logits_topk; pub mod lstm; pub mod mel_spectrogram; pub mod moe; @@ -124,7 +120,6 @@ pub mod patch_embed_mma_block_scaled; pub mod patch_unfold_qwen; pub mod pos_emb_2d_add; pub mod resize_normalize; -pub mod sampling; pub mod sdpa_bidirectional; pub mod sdpa_bidirectional_d128_relpos; pub mod sdpa_bidirectional_windowed; diff --git a/crates/metaltile-std/src/kernels/mod.rs b/crates/metaltile-std/src/kernels/mod.rs index bff78df2..28aba0ba 100644 --- a/crates/metaltile-std/src/kernels/mod.rs +++ b/crates/metaltile-std/src/kernels/mod.rs @@ -10,3 +10,4 @@ pub mod convolution; pub mod norm; pub mod rope; +pub mod sampling; diff --git a/crates/metaltile-std/src/ffai/sampling.rs b/crates/metaltile-std/src/kernels/sampling/categorical_sample.rs similarity index 95% rename from crates/metaltile-std/src/ffai/sampling.rs rename to crates/metaltile-std/src/kernels/sampling/categorical_sample.rs index c86e19e3..9fe8b85d 100644 --- a/crates/metaltile-std/src/ffai/sampling.rs +++ b/crates/metaltile-std/src/kernels/sampling/categorical_sample.rs @@ -40,7 +40,7 @@ use metaltile::kernel; // replaced by 1 × n/lsize chunk-traverse per lane + an 8-stage scan + // 1 × n/lsize finalizing walk on the winning lane. #[kernel] -pub fn softmax_categorical_sample( +pub fn mt_softmax_categorical_sample( inp: Tensor, out: Tensor, temperature_in: Tensor, @@ -136,7 +136,7 @@ pub fn softmax_categorical_sample( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::softmax_categorical_sample; + use super::mt_softmax_categorical_sample; use crate::utils::pack_f32; #[test_kernel(dtypes = [f32, f16, bf16], tol = 0.5)] @@ -148,7 +148,7 @@ pub mod kernel_tests { let (n, idx, spike) = (1024usize, 813usize, 30.0f32); let mut logits = vec![0.0f32; n]; logits[idx] = spike; - TestSetup::new(softmax_categorical_sample::kernel_ir_for(dt)) + TestSetup::new(mt_softmax_categorical_sample::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("inp", pack_f32(&logits, dt), dt)) .input(TestBuffer::zeros("out", 1, DType::U32)) @@ -160,19 +160,19 @@ pub mod kernel_tests { } } -/// New-syntax benchmark for `softmax_categorical_sample` at Qwen3 vocab +/// New-syntax benchmark for `mt_softmax_categorical_sample` at Qwen3 vocab /// scale (Reduction, single threadgroup; cooperative max + scan + CDF /// walk). Random logits, fixed temperature and uniform draw. pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::softmax_categorical_sample; + use super::mt_softmax_categorical_sample; use crate::utils::pack_f32; #[bench(dtypes = [f32, f16, bf16])] fn bench_softmax_categorical_sample(dt: DType) -> BenchSetup { let n = 152_064usize; - BenchSetup::new(softmax_categorical_sample::kernel_ir_for(dt)) + BenchSetup::new(mt_softmax_categorical_sample::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("inp", n, dt)) .buffer(BenchBuffer::zeros("out", 1, DType::U32).output()) diff --git a/crates/metaltile-std/src/ffai/logits_min_p.rs b/crates/metaltile-std/src/kernels/sampling/logits_min_p.rs similarity index 93% rename from crates/metaltile-std/src/ffai/logits_min_p.rs rename to crates/metaltile-std/src/kernels/sampling/logits_min_p.rs index 907019b0..0d956dd4 100644 --- a/crates/metaltile-std/src/ffai/logits_min_p.rs +++ b/crates/metaltile-std/src/kernels/sampling/logits_min_p.rs @@ -33,7 +33,7 @@ use metaltile::kernel; #[kernel] -pub fn logits_min_p_mask( +pub fn mt_logits_min_p_mask( inp: Tensor, out: Tensor, #[constexpr] n: u32, @@ -59,7 +59,7 @@ pub fn logits_min_p_mask( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::logits_min_p_mask; + use super::mt_logits_min_p_mask; use crate::utils::{pack_f32, unpack_f32}; /// CPU oracle: per row, keep `v` iff `exp(v − row_max) ≥ min_p`. @@ -85,7 +85,7 @@ pub mod kernel_tests { let logits: Vec = (0..n * rows).map(|i| (i % 53) as f32 * 0.2 - 5.0).collect(); let rounded = unpack_f32(&pack_f32(&logits, dt), dt); let expected = cpu_min_p_mask(&rounded, n, rows, min_p); - TestSetup::new(logits_min_p_mask::kernel_ir_for(dt)) + TestSetup::new(mt_logits_min_p_mask::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("inp", pack_f32(&logits, dt), dt)) .input(TestBuffer::zeros("out", n * rows, dt)) @@ -96,17 +96,17 @@ pub mod kernel_tests { } } -/// New-syntax benchmark for `logits_min_p_mask` at Qwen3 vocab scale +/// New-syntax benchmark for `mt_logits_min_p_mask` at Qwen3 vocab scale /// (Reduction, one TG per row). pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::logits_min_p_mask; + use super::mt_logits_min_p_mask; #[bench(dtypes = [f32, f16, bf16])] fn bench_logits_min_p_mask(dt: DType) -> BenchSetup { let (n, rows) = (152_064usize, 2usize); - BenchSetup::new(logits_min_p_mask::kernel_ir_for(dt)) + BenchSetup::new(mt_logits_min_p_mask::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("inp", n * rows, dt)) .buffer(BenchBuffer::zeros("out", n * rows, dt).output()) diff --git a/crates/metaltile-std/src/ffai/logits_processors.rs b/crates/metaltile-std/src/kernels/sampling/logits_processors.rs similarity index 94% rename from crates/metaltile-std/src/ffai/logits_processors.rs rename to crates/metaltile-std/src/kernels/sampling/logits_processors.rs index 99240a61..aefc3e3c 100644 --- a/crates/metaltile-std/src/ffai/logits_processors.rs +++ b/crates/metaltile-std/src/kernels/sampling/logits_processors.rs @@ -52,7 +52,7 @@ use metaltile::kernel; // read/write out of bounds; the runtime should size the dispatch so the // total thread count exactly matches the logits length. #[kernel] -pub fn logits_temperature(inp: Tensor, out: Tensor, #[constexpr] temperature: f32) { +pub fn mt_logits_temperature(inp: Tensor, out: Tensor, #[constexpr] temperature: f32) { let i = program_id::<0>(); let inv_t = 1.0f32 / temperature; let v = load(inp[i]).cast::(); @@ -89,7 +89,7 @@ pub fn logits_temperature(inp: Tensor, out: Tensor, #[constexpr] temper // - **No `threadgroup_*` / `simd_*` cooperation** — every thread is // independent. The only invariant is the dedupe contract above. #[kernel] -pub fn logits_repetition_penalty( +pub fn mt_logits_repetition_penalty( mut logits: Tensor, token_ids: Tensor, #[constexpr] penalty: f32, @@ -104,7 +104,7 @@ pub fn logits_repetition_penalty( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::{logits_repetition_penalty, logits_temperature}; + use super::{mt_logits_repetition_penalty, mt_logits_temperature}; use crate::utils::{pack_f32, unpack_f32}; fn u32_bytes(v: &[u32]) -> Vec { v.iter().flat_map(|x| x.to_le_bytes()).collect() } @@ -118,7 +118,7 @@ pub mod kernel_tests { // Kernel computes `v * (1/T)` in f32 then casts back; oracle mirrors. let inv_t = 1.0f32 / temperature; let expected: Vec = rounded.iter().map(|&v| v * inv_t).collect(); - TestSetup::new(logits_temperature::kernel_ir_for(dt)) + TestSetup::new(mt_logits_temperature::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("inp", pack_f32(&logits, dt), dt)) .input(TestBuffer::zeros("out", n, dt)) @@ -145,7 +145,7 @@ pub mod kernel_tests { let v = expected[tok as usize]; expected[tok as usize] = if v > 0.0 { v / penalty } else { v * penalty }; } - TestSetup::new(logits_repetition_penalty::kernel_ir_for(dt)) + TestSetup::new(mt_logits_repetition_penalty::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("logits", pack_f32(&logits, dt), dt)) .input(TestBuffer::from_vec("token_ids", u32_bytes(&token_ids), DType::U32)) @@ -159,7 +159,7 @@ pub mod kernel_tests { pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::{logits_repetition_penalty, logits_temperature}; + use super::{mt_logits_repetition_penalty, mt_logits_temperature}; fn u32_bytes(v: impl Iterator) -> Vec { v.flat_map(|x| x.to_le_bytes()).collect() @@ -168,7 +168,7 @@ pub mod kernel_benches { #[bench(dtypes = [f32, f16, bf16])] fn bench_logits_temperature(dt: DType) -> BenchSetup { let n = 152_064usize; - BenchSetup::new(logits_temperature::kernel_ir_for(dt)) + BenchSetup::new(mt_logits_temperature::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .buffer(BenchBuffer::random("inp", n, dt)) .buffer(BenchBuffer::zeros("out", n, dt).output()) @@ -182,7 +182,7 @@ pub mod kernel_benches { // A modest context window of distinct token ids over a Qwen-scale // vocab — one thread per token id. let (vocab, n_tokens) = (152_064usize, 2048usize); - BenchSetup::new(logits_repetition_penalty::kernel_ir_for(dt)) + BenchSetup::new(mt_logits_repetition_penalty::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .buffer(BenchBuffer::random("logits", vocab, dt).output()) .buffer(BenchBuffer::from_vec( diff --git a/crates/metaltile-std/src/ffai/logits_top_p.rs b/crates/metaltile-std/src/kernels/sampling/logits_top_p.rs similarity index 96% rename from crates/metaltile-std/src/ffai/logits_top_p.rs rename to crates/metaltile-std/src/kernels/sampling/logits_top_p.rs index 71c835e4..bd8ea86a 100644 --- a/crates/metaltile-std/src/ffai/logits_top_p.rs +++ b/crates/metaltile-std/src/kernels/sampling/logits_top_p.rs @@ -42,7 +42,7 @@ use metaltile::kernel; #[kernel] -pub fn logits_top_p_mask( +pub fn mt_logits_top_p_mask( inp: Tensor, out: Tensor, #[constexpr] n: u32, @@ -102,7 +102,7 @@ pub fn logits_top_p_mask( pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::logits_top_p_mask; + use super::mt_logits_top_p_mask; use crate::utils::{pack_f32, unpack_f32}; /// Bisection halvings — must match the kernel's loop bound. @@ -142,7 +142,7 @@ pub mod kernel_tests { let logits: Vec = (0..n * rows).map(|i| (i % 53) as f32 * 0.2 - 5.0).collect(); let rounded = unpack_f32(&pack_f32(&logits, dt), dt); let expected = cpu_top_p_mask(&rounded, n, rows, top_p); - TestSetup::new(logits_top_p_mask::kernel_ir_for(dt)) + TestSetup::new(mt_logits_top_p_mask::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .input(TestBuffer::from_vec("inp", pack_f32(&logits, dt), dt)) .input(TestBuffer::zeros("out", n * rows, dt)) @@ -153,17 +153,17 @@ pub mod kernel_tests { } } -/// New-syntax benchmark for `logits_top_p_mask` at Qwen3 vocab scale +/// New-syntax benchmark for `mt_logits_top_p_mask` at Qwen3 vocab scale /// (Reduction, one TG per row; 24 bisection passes over the row). pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::logits_top_p_mask; + use super::mt_logits_top_p_mask; #[bench(dtypes = [f32, f16, bf16])] fn bench_logits_top_p_mask(dt: DType) -> BenchSetup { let (n, rows) = (152_064usize, 2usize); - BenchSetup::new(logits_top_p_mask::kernel_ir_for(dt)) + BenchSetup::new(mt_logits_top_p_mask::kernel_ir_for(dt)) .mode(KernelMode::Reduction) .buffer(BenchBuffer::random("inp", n * rows, dt)) .buffer(BenchBuffer::zeros("out", n * rows, dt).output()) diff --git a/crates/metaltile-std/src/ffai/logits_topk.rs b/crates/metaltile-std/src/kernels/sampling/logits_topk.rs similarity index 92% rename from crates/metaltile-std/src/ffai/logits_topk.rs rename to crates/metaltile-std/src/kernels/sampling/logits_topk.rs index e7aab724..9a3dbe54 100644 --- a/crates/metaltile-std/src/ffai/logits_topk.rs +++ b/crates/metaltile-std/src/kernels/sampling/logits_topk.rs @@ -38,7 +38,7 @@ use metaltile::kernel; #[kernel] -pub fn logits_topk_mask(inp: Tensor, out: Tensor, #[constexpr] threshold: f32) { +pub fn mt_logits_topk_mask(inp: Tensor, out: Tensor, #[constexpr] threshold: f32) { let i = program_id::<0>(); let v = load(inp[i]).cast::(); // `select(cond, lhs, rhs)` returns lhs when cond is true. @@ -51,7 +51,7 @@ pub fn logits_topk_mask(inp: Tensor, out: Tensor, #[constexpr] threshol pub mod kernel_tests { use metaltile::{test::*, test_kernel}; - use super::logits_topk_mask; + use super::mt_logits_topk_mask; use crate::utils::{pack_f32, unpack_f32}; /// K-th largest value (descending) — how callers pre-compute the cutoff. @@ -72,7 +72,7 @@ pub mod kernel_tests { let threshold = kth_largest(&rounded, k); let expected: Vec = rounded.iter().map(|&v| if v >= threshold { v } else { f32::NEG_INFINITY }).collect(); - TestSetup::new(logits_topk_mask::kernel_ir_for(dt)) + TestSetup::new(mt_logits_topk_mask::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .input(TestBuffer::from_vec("inp", pack_f32(&logits, dt), dt)) .input(TestBuffer::zeros("out", n, dt)) @@ -82,17 +82,17 @@ pub mod kernel_tests { } } -/// New-syntax benchmark for `logits_topk_mask` at Qwen3 vocab scale +/// New-syntax benchmark for `mt_logits_topk_mask` at Qwen3 vocab scale /// (Grid3D, one thread per vocab position). pub mod kernel_benches { use metaltile::{bench, test::*}; - use super::logits_topk_mask; + use super::mt_logits_topk_mask; #[bench(dtypes = [f32, f16, bf16])] fn bench_logits_topk_mask(dt: DType) -> BenchSetup { let n = 152_064usize; - BenchSetup::new(logits_topk_mask::kernel_ir_for(dt)) + BenchSetup::new(mt_logits_topk_mask::kernel_ir_for(dt)) .mode(KernelMode::Grid3D) .buffer(BenchBuffer::random("inp", n, dt)) .buffer(BenchBuffer::zeros("out", n, dt).output()) diff --git a/crates/metaltile-std/src/kernels/sampling/mod.rs b/crates/metaltile-std/src/kernels/sampling/mod.rs new file mode 100644 index 00000000..506a1413 --- /dev/null +++ b/crates/metaltile-std/src/kernels/sampling/mod.rs @@ -0,0 +1,15 @@ +//! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric +//! SPDX-License-Identifier: Apache-2.0 +//! Sampling kernels — the sampling family (see +//! `docs/specs/KERNEL_CONSOLIDATION_PLAN.md`): the logits→token pipeline. +//! Softmax and sort (from the legacy `mlx/`), the fused categorical sampler, +//! and the logits processors / masks (top-k, top-p, min-p, temperature, +//! repetition penalty) from the legacy `ffai/`. + +pub mod categorical_sample; +pub mod logits_min_p; +pub mod logits_processors; +pub mod logits_top_p; +pub mod logits_topk; +pub mod softmax; +pub mod sort; diff --git a/crates/metaltile-std/src/mlx/softmax.rs b/crates/metaltile-std/src/kernels/sampling/softmax.rs similarity index 100% rename from crates/metaltile-std/src/mlx/softmax.rs rename to crates/metaltile-std/src/kernels/sampling/softmax.rs diff --git a/crates/metaltile-std/src/mlx/sort.rs b/crates/metaltile-std/src/kernels/sampling/sort.rs similarity index 100% rename from crates/metaltile-std/src/mlx/sort.rs rename to crates/metaltile-std/src/kernels/sampling/sort.rs diff --git a/crates/metaltile-std/src/mlx/mod.rs b/crates/metaltile-std/src/mlx/mod.rs index 7f1c4b8e..2f898776 100644 --- a/crates/metaltile-std/src/mlx/mod.rs +++ b/crates/metaltile-std/src/mlx/mod.rs @@ -52,8 +52,6 @@ pub mod scan; pub mod scatter_axis; pub mod sdpa_vector; pub mod sgload_smoke; -pub mod softmax; -pub mod sort; pub mod steel; pub mod strided; pub mod swiglu; diff --git a/crates/metaltile-std/tests/kernel_registry_consistency.rs b/crates/metaltile-std/tests/kernel_registry_consistency.rs index c8902c68..5018295e 100644 --- a/crates/metaltile-std/tests/kernel_registry_consistency.rs +++ b/crates/metaltile-std/tests/kernel_registry_consistency.rs @@ -322,7 +322,7 @@ fn no_undefined_mt_symbols_in_emitted_msl() { #[test] fn kernel_annotations_have_matching_inventory_submit() { // Collect kernel-named registry entries (kernel_name → matched-via-name) - // PLUS kernel_ir function paths (e.g. `softmax_categorical_sample::kernel_ir_for`) + // PLUS kernel_ir function paths (e.g. `mt_softmax_categorical_sample::kernel_ir_for`) // so we can match either form. The macro expands `#[kernel]` to // give each function a `kernel_ir_for(dt) -> Kernel` associated // function inside a module of the same name; `inventory::submit!` diff --git a/crates/metaltile-std/tests/softmax_categorical_sample_matrix.rs b/crates/metaltile-std/tests/softmax_categorical_sample_matrix.rs index 42022d15..0f446c7f 100644 --- a/crates/metaltile-std/tests/softmax_categorical_sample_matrix.rs +++ b/crates/metaltile-std/tests/softmax_categorical_sample_matrix.rs @@ -1,6 +1,6 @@ //! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric //! SPDX-License-Identifier: Apache-2.0 -//! Matrix coverage for `ffai::softmax_categorical_sample` — +//! Matrix coverage for `ffai::mt_softmax_categorical_sample` — //! dtype × vocab × temperature × distribution × uniform-draw, plus //! invariant property assertions that catch silent codegen bugs //! numeric drift alone can't detect. @@ -61,7 +61,7 @@ use std::collections::BTreeMap; use common::{Dt, gpu_lock, pack_bytes}; use metaltile::{Context, core::ir::KernelMode}; -use metaltile_std::ffai::sampling::softmax_categorical_sample; +use metaltile_std::kernels::sampling::categorical_sample::mt_softmax_categorical_sample; // ── CPU oracle (mirrors the kernel's three-pass shape) ──────────────── @@ -102,7 +102,7 @@ fn run_sample(ctx: &Context, dt: Dt, logits: &[f32], temperature: f32, uniform: buffers.insert("uniform_in".into(), uniform.to_le_bytes().to_vec()); buffers.insert("n".into(), (n as u32).to_le_bytes().to_vec()); - let mut kernel = softmax_categorical_sample::kernel_ir_for(dt.to_dtype()); + let mut kernel = mt_softmax_categorical_sample::kernel_ir_for(dt.to_dtype()); kernel.mode = KernelMode::Reduction; // Fixed TPG = 256 per kernel invariant (tg_max + tg_sum both 256-wide, diff --git a/crates/metaltile-std/tests/softmax_categorical_sample_perf.rs b/crates/metaltile-std/tests/softmax_categorical_sample_perf.rs index ea42e9d1..4aeac778 100644 --- a/crates/metaltile-std/tests/softmax_categorical_sample_perf.rs +++ b/crates/metaltile-std/tests/softmax_categorical_sample_perf.rs @@ -1,6 +1,6 @@ //! Copyright 2026 0xClandestine, Ekryski, TheTom, Ambisphaeric //! SPDX-License-Identifier: Apache-2.0 -//! Quick perf-only timing for `softmax_categorical_sample` at vocab=152K. +//! Quick perf-only timing for `mt_softmax_categorical_sample` at vocab=152K. //! //! Ignored by default — run with `--ignored` to measure. Times 1000 //! sequential dispatches on the same buffers and prints the median + @@ -19,7 +19,7 @@ use metaltile::{ Context, core::{dtype::DType, ir::KernelMode}, }; -use metaltile_std::ffai::sampling::softmax_categorical_sample; +use metaltile_std::kernels::sampling::categorical_sample::mt_softmax_categorical_sample; #[test] #[ignore] @@ -38,7 +38,7 @@ fn perf_softmax_categorical_sample_vocab_152k() { buffers.insert("n".into(), (n as u32).to_le_bytes().to_vec()); let ctx = Context::new().expect("Context::new on macOS"); - let mut kernel = softmax_categorical_sample::kernel_ir_for(DType::F32); + let mut kernel = mt_softmax_categorical_sample::kernel_ir_for(DType::F32); kernel.mode = KernelMode::Reduction; // Warmup. @@ -58,7 +58,7 @@ fn perf_softmax_categorical_sample_vocab_152k() { let median_ns = samples[samples.len() / 2]; let min_ns = samples[0]; println!( - "softmax_categorical_sample vocab={n}: median={:.1}µs min={:.1}µs", + "mt_softmax_categorical_sample vocab={n}: median={:.1}µs min={:.1}µs", median_ns as f64 / 1000.0, min_ns as f64 / 1000.0, ); diff --git a/docs/specs/KERNEL_AUDIT.md b/docs/specs/KERNEL_AUDIT.md index a8b0633e..20c24f60 100644 --- a/docs/specs/KERNEL_AUDIT.md +++ b/docs/specs/KERNEL_AUDIT.md @@ -72,9 +72,9 @@ Local verification of NAX kernels is the developer's responsibility on M4+ hardw | swiglu (`silu(gate)·up` fused MLP activation) | ✗ | ✗ | ✓ | `mlx/swiglu.rs` → `mt_swiglu`. Standard modern-transformer MLP activation (Llama 4, Qwen3 dense + MoE, Gemma, Mistral). | | random (key hash → u32) | ✓ | ✓ | ✓ | `mlx/random.rs` → `mt_random_hash`. | | reduce (sum/prod/max/min — all + row + col) | ✓ | ✓ | ✓ | `mlx/reduce.rs` covers `all_reduce*`, `row_reduce*`, `col_reduce*`, and `seg_reduce*` (Grid3D one-thread-per-segment, contiguous fixed-length runs) — all four ops for each shape. | -| sort | ✓ | ✓ | ✓ | `mlx/sort.rs` → `mt_sort` (single-block bitonic) + `mt_merge` (multi-block merge) + `mt_sort_segmented` (per-row bitonic for `[batch, n]` matrices, `n ≤ 1024`, one TG per row). | +| sort | ✓ | ✓ | ✓ | `kernels/sampling/sort.rs` → `mt_sort` (single-block bitonic) + `mt_merge` (multi-block merge) + `mt_sort_segmented` (per-row bitonic for `[batch, n]` matrices, `n ≤ 1024`, one TG per row). | | scan (prefix sum/prod/max/min) | ✓ | ✓ | ✓ | `mlx/scan.rs` → `mt_scan` + `mt_scan_exclusive` (sum), `mt_scan_prod` / `mt_scan_max` / `mt_scan_min` + exclusive variants. Sum pair uses hardware `simd_scan_exclusive`; the prod/max/min pairs use a `tgs[lsize]` threadgroup buffer for sequential cross-thread prefix reads. | -| softmax | ✓ | ✓ | ✓ | `mlx/softmax.rs` → `mt_softmax` (looped + single-row collapsed). | +| softmax | ✓ | ✓ | ✓ | `kernels/sampling/softmax.rs` → `mt_softmax` (looped + single-row collapsed). | | logsumexp | ✓ | ✓ | ✓ | `mlx/logsumexp.rs` → `mt_logsumexp`. | | layer_norm | ✓ | ✓ | ✓ | `kernels/norm/layer_norm.rs` → `mt_layer_norm`. | | rms_norm | ✓ | ✓ | ✓ | `kernels/norm/rms_norm.rs` → `mt_rms_norm` + `mt_rms_norm_small` (2-elem/thread, small-head_dim per-head q_norm/k_norm) + `mt_rms_norm_wide` (strided wide-row variant for `head_dim > 4096`, e.g. Gemma 4 31B hidden=5376). | @@ -138,8 +138,8 @@ Local verification of NAX kernels is the developer's responsibility on M4+ hardw | batched_qkv_qgemv (Q/K/V 4-bit qGEMV → 1 dispatch) | ✗ | ✓ | ✓ | `ffai/batched_qkv_qgemv.rs` → `ffai_batched_qkv_qgemv` (one-row-per-TG) + `ffai_batched_qkv_qgemv_fast` (8-row-per-TG, GQA-guarded, PR #154). `program_id::<2>()` selects Q/K/V, output concatenated `[Q\|K\|V]`. | | kv_cache_update (raw bf16/fp16 single-token append) | ✗ | ✗ | ✓ | `ffai/kv_cache.rs` → `kv_cache_update`. FFAI-only; raw cache append. | | kv_cache (affine-quant int4/int8/fp8 quantize + bulk dequant) | ~ | ~ | ✓ | `ffai/kv_cache.rs` — `quantize_kv` + `bulk_dequant_kv` for int4/int8. **fp8** (PR #157): `quantize_kv_fp8_{e4m3,e5m2}` + `bulk_dequant_kv_fp8_{e4m3,e5m2}`. Per-group amax → scale quantize, byte-shift extract + biased-exp decode. E4M3: mantissa_bits=3, e_bias=-6, max=448; E5M2: mantissa_bits=2, e_bias=-14, max=57344. Closes the host-side fp8 KV round-trip. | -| sampling (softmax + categorical inverse-CDF) | ✗ | ✗ | ✓ | `ffai/sampling.rs` → `softmax_categorical_sample`. Companion to `ffai_argmax` for `T > 0` decode. | -| logits processors (temperature, repetition penalty, top-k / top-p / min-p masks) | ✗ | ✗ | ✓ | `ffai/logits_{processors,topk,top_p,min_p}.rs` — in-place decode-form sampler stages composed before `softmax_categorical_sample`. FFAI-only. | +| sampling (softmax + categorical inverse-CDF) | ✗ | ✗ | ✓ | `kernels/sampling/categorical_sample.rs` → `mt_softmax_categorical_sample`. Companion to `ffai_argmax` for `T > 0` decode. | +| logits processors (temperature, repetition penalty, top-k / top-p / min-p masks) | ✗ | ✗ | ✓ | `kernels/sampling/logits_{processors,topk,top_p,min_p}.rs` — in-place decode-form sampler stages composed before `mt_softmax_categorical_sample`. | | sdpa_decode + learned attention sink (GPT-OSS-20B) | ✗ | ~ | ✓ | `ffai/sdpa_decode.rs` `has_sink` / `sink_logit` constexprs. GPT-OSS-20B's per-head learned attention-sink logit folds into the cross-simdgroup softmax denominator on-GPU as a virtual key — removing the host-side post-hoc rescale that previously cost a CPU sync per attention layer. | | gated_rmsnorm (fp32-in gated RMSNorm → activation dtype) | ✗ | ✗ | ✓ | `kernels/norm/gated_rmsnorm.rs` → `mt_gated_rmsnorm`. Fused Qwen3.5 / 3.6 GDN post-step `out = w·rmsNorm(y)·silu(z)`; `y` arrives fp32 (the `gated_delta` recurrence output). Closes the per-GDN-layer host-side CPU sync (~75 % of Qwen3.5/3.6 layers). | | conv2d (vision patch conv — im2col + tiled GEMM) | ✓ | ✓ | ✓ | `ffai/conv2d.rs` → `conv2d_patch14` / `conv2d_patch16` + `conv2d_generic`. NCHW input, OIHW weight; direct conv (implicit im2col, one thread per output). VLM front-end. | diff --git a/docs/specs/KERNEL_CONSOLIDATION_PLAN.md b/docs/specs/KERNEL_CONSOLIDATION_PLAN.md index 6bdc6d0c..d4fcd40d 100644 --- a/docs/specs/KERNEL_CONSOLIDATION_PLAN.md +++ b/docs/specs/KERNEL_CONSOLIDATION_PLAN.md @@ -12,13 +12,13 @@ Where they overlap (naming, per-file shape), defer to the style guide. > **Status:** in progress. `convolution/` is the proven exemplar — its > *structural* consolidation has landed (move, dedup, per-format macro DRY-ing), -> and `rope/` + `norm/` have landed too. Two optimization phases remain open for -> every family: replace the interim `macro_rules!` with `variants(...)`, and -> merge the `*_block_scaled` / `*_mma` files into the dimensionality file (§4). -> "✅ done" in §6 means the structural move is in; the optimization phases are -> tracked separately. The `*_block_scaled_qgemv` format matrices moved with -> `norm/` but still await the format-axis fold (§7). The remaining families -> migrate one-per-PR per §6. +> and `rope/` + `norm/` + `sampling/` have landed too. Two optimization phases +> remain open for every family: replace the interim `macro_rules!` with +> `variants(...)`, and merge the `*_block_scaled` / `*_mma` files into the +> dimensionality file (§4). "✅ done" in §6 means the structural move is in; the +> optimization phases are tracked separately. The `*_block_scaled_qgemv` format +> matrices moved with `norm/` but still await the format-axis fold (§7). The +> remaining families migrate one-per-PR per §6. ## 1. Why @@ -60,7 +60,7 @@ crates/metaltile-std/src/kernels/ audio/ mel_spectrogram(+magnitude/stft/filterbank) · lstm · vocoder · snake1d · upsample vision/ resize_normalize(+bicubic) · im2col · patch_unfold · pos_emb_2d · avg_pool2d · transpose_th · frame_diff · broadcast_affine - sampling/ logits_topk/top_p/min_p/processors · sampling · softmax · sort + sampling/ ✅ DONE — logits_topk/top_p/min_p/processors · categorical_sample · softmax · sort kv_cache/ kv_cache(_update_many) · fft primitives.rs cross-family decode/reduce ops (mt_decode_e2m1/e4m3/e5m2/e8m0, mt_unpack_nbit, …) mod.rs pub mod ops; pub mod gemm; pub mod sdpa; … @@ -156,8 +156,8 @@ payoff last: | Wave | Families | Rationale | Payoff | |---|---|---|---| -| ✅ done | `convolution/`, `rope/`, `norm/` | exemplar + wave-1 families | 24k → ~1.6k | -| 1 | `sampling/`, `ops/` | self-contained, mostly elementwise / few formats | small, sets the pattern | +| ✅ done | `convolution/`, `rope/`, `norm/`, `sampling/` | exemplar + wave-1 families | 24k → ~1.6k | +| 1 | `ops/` | self-contained, mostly elementwise / few formats | small, sets the pattern | | 2 | `gemm/`, `ssm/`, `audio/`, `vision/`, `kv_cache/` | moderate size, few cross-deps | medium | | 3 | `sdpa/`, `moe/`, **`quant/`** | hardest axes (head-dim d64..d512; bm8/bm64×int8; the 30-format matrix) — most of the ~150k LOC | the bulk |