From 26e4e473e371fc13e190021118e47b809b9bad4a Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 4 Jun 2026 03:13:00 +0000 Subject: [PATCH 1/6] draft Signed-off-by: Isotr0py --- src/vision/image_processor.rs | 10 + src/vision/processors/minimax_m3.rs | 489 ++++++++++++++++++++++++++++ src/vision/processors/mod.rs | 3 + 3 files changed, 502 insertions(+) create mode 100644 src/vision/processors/minimax_m3.rs diff --git a/src/vision/image_processor.rs b/src/vision/image_processor.rs index 97cc870..cf3272f 100644 --- a/src/vision/image_processor.rs +++ b/src/vision/image_processor.rs @@ -474,6 +474,16 @@ impl ImageProcessorRegistry { Box::new(super::processors::KimiK25Processor::new()), ); + // Register MiniMax-M3 VL + registry.register( + "minimax-m3", + Box::new(super::processors::MiniMaxM3Processor::new()), + ); + registry.register( + "minimax_m3", + Box::new(super::processors::MiniMaxM3Processor::new()), + ); + registry } } diff --git a/src/vision/processors/minimax_m3.rs b/src/vision/processors/minimax_m3.rs new file mode 100644 index 0000000..597ffd2 --- /dev/null +++ b/src/vision/processors/minimax_m3.rs @@ -0,0 +1,489 @@ +//! MiniMax-M3 VL image processor. +//! +//! Ported from HuggingFace `MiniMaxM3VLImageProcessor`. The model documents this +//! as "Copied from Qwen2VLImageProcessorFast with resize changed to vLLM style": +//! the patchify pipeline (rescale → normalize → reshape into +//! `[grid_t, grid_h, grid_w, ...]` patches) is identical to Qwen2-VL, so we reuse +//! [`QwenVLProcessorBase`] for it. The only difference is the resize step. +//! +//! # vLLM-style resize (`get_hw_multiple_of`) +//! +//! Unlike Qwen's smart-resize (which targets a min/max *pixel* budget), MiniMax: +//! +//! 1. Rounds each dimension **up** to a multiple of `patch_size * merge_size`. +//! 2. If either dimension exceeds `max_size` (width, height), scales the image +//! down to fit while preserving aspect ratio, then re-aligns (rounds up) to +//! the factor. +//! +//! There is no lower (min-pixels) bound. `max_size` is inferred from the +//! processor's `size` (default `{height: 672, width: 672}`) and must itself be +//! divisible by the factor. + +use image::{imageops::FilterType, DynamicImage, GenericImageView}; + +use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase}; +use crate::vision::{ + image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, + preprocessor_config::PreProcessorConfig, + transforms::{pil_to_filter, resize, to_tensor, to_tensor_and_normalize, TransformError}, +}; + +/// CLIP normalization mean values used by MiniMax-M3 VL. +pub const MINIMAX_M3_MEAN: [f64; 3] = [0.48145466, 0.4578275, 0.40821073]; + +/// CLIP normalization std values used by MiniMax-M3 VL. +pub const MINIMAX_M3_STD: [f64; 3] = [0.26862954, 0.26130258, 0.27577711]; + +/// Default vision encoder patch size. +pub const DEFAULT_PATCH_SIZE: usize = 14; + +/// Default spatial merge size (token reduction). +pub const DEFAULT_MERGE_SIZE: usize = 2; + +/// Default temporal patch size (for video frames; images repeat the single frame). +pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; + +/// Default vLLM-style resize bound as `(max_width, max_height)`. +/// +/// Inferred from the HF default `size = {"height": 672, "width": 672}`. Must be +/// divisible by the factor (`patch_size * merge_size`); 672 / 28 = 24. +pub const DEFAULT_MAX_SIZE: (usize, usize) = (672, 672); + +/// Round `x` up to the nearest multiple of `multiple`. +#[inline] +fn ceil_to_multiple(x: usize, multiple: usize) -> usize { + if multiple == 0 || x % multiple == 0 { + x + } else { + x + (multiple - x % multiple) + } +} + +/// MiniMax-M3 VL image processor. +/// +/// Wraps [`QwenVLProcessorBase`] for the shared patchify/grid logic and overrides +/// the resize with the vLLM-style [`Self::vllm_resize`]. +#[derive(Debug, Clone)] +pub struct MiniMaxM3Processor { + inner: QwenVLProcessorBase, + /// vLLM-style resize bound as `(max_width, max_height)`. + max_size: (usize, usize), +} + +impl Default for MiniMaxM3Processor { + fn default() -> Self { + Self::new() + } +} + +impl MiniMaxM3Processor { + /// Create a new MiniMax-M3 processor with default settings. + /// + /// Defaults: + /// - patch_size: 14 + /// - merge_size: 2 + /// - temporal_patch_size: 2 + /// - max_size: (672, 672) + /// - normalization: CLIP mean/std + pub fn new() -> Self { + Self::with_config( + DEFAULT_PATCH_SIZE, + DEFAULT_MERGE_SIZE, + DEFAULT_TEMPORAL_PATCH_SIZE, + DEFAULT_MAX_SIZE, + ) + } + + /// Create a processor with custom settings. + pub fn with_config( + patch_size: usize, + merge_size: usize, + temporal_patch_size: usize, + max_size: (usize, usize), + ) -> Self { + Self { + // min_pixels / max_pixels are unused: MiniMax never calls smart_resize. + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size, + merge_size, + min_pixels: 0, + max_pixels: usize::MAX, + temporal_patch_size, + mean: MINIMAX_M3_MEAN, + std: MINIMAX_M3_STD, + model_name: "minimax-m3", + }), + max_size, + } + } + + /// Create a processor from a HuggingFace preprocessor config. + pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { + let patch_size = config.get_patch_size(DEFAULT_PATCH_SIZE); + let merge_size = config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE); + let temporal_patch_size = config + .temporal_patch_size + .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE); + // `get_target_size` returns (height, width); max_size is (width, height). + let max_size = config + .get_target_size() + .map(|(h, w)| (w as usize, h as usize)) + .unwrap_or(DEFAULT_MAX_SIZE); + Self::with_config(patch_size, merge_size, temporal_patch_size, max_size) + } + + /// Get the patch size. + pub fn patch_size(&self) -> usize { + self.inner.patch_size() + } + + /// Get the merge size. + pub fn merge_size(&self) -> usize { + self.inner.merge_size() + } + + /// Get the temporal patch size. + pub fn temporal_patch_size(&self) -> usize { + self.inner.temporal_patch_size() + } + + /// Get the vLLM-style resize bound as `(max_width, max_height)`. + pub fn max_size(&self) -> (usize, usize) { + self.max_size + } + + /// Get the factor for dimension alignment (`patch_size * merge_size`). + #[inline] + pub fn get_factor(&self) -> usize { + self.inner.get_factor() + } + + /// Compute the target `(new_width, new_height)`, both multiples of `factor`, + /// scaled to fit within `max_size`. Mirrors HF `get_hw_multiple_of` for the + /// `(max_w, max_h)` tuple case (the only one MiniMax uses). + fn get_hw_multiple_of(&self, width: usize, height: usize, factor: usize) -> (usize, usize) { + let (max_w, max_h) = self.max_size; + let mut new_w = ceil_to_multiple(width, factor); + let mut new_h = ceil_to_multiple(height, factor); + + if new_w > max_w || new_h > max_h { + // Scale down to fit within max_size while maintaining aspect ratio. + // (new_w * max_w) // new_w == max_w, kept explicit to match HF. + let new_w_ = max_w.min(new_w * max_h / new_h); + let new_h_ = (new_h * max_w / new_w).min(max_h); + // Re-align (round up) to the factor. + new_w = ceil_to_multiple(new_w_, factor); + new_h = ceil_to_multiple(new_h_, factor); + } + + (new_w, new_h) + } + + /// vLLM-style resize. Returns `(new_height, new_width)`, both multiples of the + /// alignment factor and bounded by `max_size`. + pub fn vllm_resize(&self, height: usize, width: usize) -> (usize, usize) { + let factor = self.get_factor(); + let (new_w, new_h) = self.get_hw_multiple_of(width, height, factor); + (new_h, new_w) + } + + /// Calculate the grid dimensions `(grid_t, grid_h, grid_w)` for an image. + pub fn calculate_grid_thw( + &self, + height: usize, + width: usize, + num_frames: usize, + ) -> (usize, usize, usize) { + self.inner.calculate_grid_thw(height, width, num_frames) + } + + /// Calculate the number of image tokens after merge. + pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize { + self.inner + .calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + /// Pick the resampling filter, defaulting to BICUBIC (the HF default) when the + /// config doesn't specify one. + fn resize_filter(config: &PreProcessorConfig) -> FilterType { + match config.resampling { + Some(r) => pil_to_filter(Some(r)), + None => FilterType::CatmullRom, // BICUBIC + } + } +} + +impl ImagePreProcessor for MiniMaxM3Processor { + fn default_mean(&self) -> [f64; 3] { + MINIMAX_M3_MEAN + } + + fn default_std(&self) -> [f64; 3] { + MINIMAX_M3_STD + } + + fn preprocess( + &self, + images: &[DynamicImage], + config: &PreProcessorConfig, + ) -> Result { + if images.is_empty() { + return Err(TransformError::EmptyBatch); + } + + let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); + + let mean = config.get_image_mean(); + let std = config.get_image_std(); + let filter = Self::resize_filter(config); + + let patch_size = self.patch_size(); + let temporal_patch_size = self.temporal_patch_size(); + let patch_features = 3 * temporal_patch_size * patch_size * patch_size; + + let mut all_patches: Vec = Vec::new(); + let mut patches_per_image: Vec = Vec::with_capacity(images.len()); + let mut grid_thw_data = Vec::with_capacity(images.len() * 3); + let mut num_img_tokens = Vec::with_capacity(images.len()); + + for image in images { + let (w, h) = image.dimensions(); + let (target_h, target_w) = self.vllm_resize(h as usize, w as usize); + + // Resize to the image's own target size (skip if dimensions match). + let (tw32, th32) = (target_w as u32, target_h as u32); + let needs_resize = config.do_resize.unwrap_or(true) && (w != tw32 || h != th32); + let resized; + let img_ref = if needs_resize { + resized = resize(image, tw32, th32, filter); + &resized + } else { + image + }; + + // Grid dimensions based on the target size (T=1 for a single image). + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1); + grid_thw_data.push(grid_t as i64); + grid_thw_data.push(grid_h as i64); + grid_thw_data.push(grid_w as i64); + + let num_patches = grid_t * grid_h * grid_w; + num_img_tokens.push(self.calculate_tokens_from_grid(grid_t, grid_h, grid_w)); + + // Convert to tensor [C, H, W] (+ normalize) in one fused pass. + let tensor = if config.do_normalize.unwrap_or(true) { + to_tensor_and_normalize(img_ref, &mean, &std) + } else { + to_tensor(img_ref) + }; + + // Patchify directly into the shared buffer. The single image frame is + // repeated across `temporal_patch_size` inside `patchify_into`, which + // matches HF's "repeat last frame to fill the temporal dim". + self.inner + .patchify_into(&tensor, grid_t, grid_h, grid_w, &mut all_patches)?; + patches_per_image.push(num_patches as i64); + } + + let total_patches: usize = patches_per_image.iter().map(|&n| n as usize).sum(); + let pixel_values = + ndarray::Array2::from_shape_vec((total_patches, patch_features), all_patches).map_err( + |e| { + TransformError::ShapeError(format!( + "Failed to create patchified pixel_values [{total_patches}, {patch_features}]: {e}" + )) + }, + )?; + + let result = + PreprocessedImages::new_dynamic(pixel_values.into_dyn(), num_img_tokens, image_sizes) + .with_extra( + "image_grid_thw", + ModelSpecificValue::int_2d(grid_thw_data, images.len(), 3), + ) + .with_extra( + "patches_per_image", + ModelSpecificValue::int_1d(patches_per_image), + ); + + Ok(result) + } + + fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { + let (new_height, new_width) = self.vllm_resize(height as usize, width as usize); + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1); + self.calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + fn model_name(&self) -> &'static str { + "minimax-m3" + } + + fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { + // Dynamic resolution: no fixed output size. + None + } +} + +#[cfg(test)] +mod tests { + use image::{Rgb, RgbImage}; + + use super::*; + use crate::vision::image_processor::ModelSpecificValue; + + fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { + DynamicImage::from(RgbImage::from_pixel(width, height, color)) + } + + #[test] + fn test_defaults() { + let p = MiniMaxM3Processor::new(); + assert_eq!(p.patch_size(), 14); + assert_eq!(p.merge_size(), 2); + assert_eq!(p.temporal_patch_size(), 2); + assert_eq!(p.get_factor(), 28); // 14 * 2 + assert_eq!(p.max_size(), (672, 672)); + } + + #[test] + fn test_mean_std() { + let p = MiniMaxM3Processor::new(); + assert_eq!(p.default_mean(), MINIMAX_M3_MEAN); + assert_eq!(p.default_std(), MINIMAX_M3_STD); + } + + #[test] + fn test_model_name() { + assert_eq!(MiniMaxM3Processor::new().model_name(), "minimax-m3"); + } + + #[test] + fn test_resize_within_bounds_aligns_up() { + let p = MiniMaxM3Processor::new(); + // 100x100 -> ceil to 28 multiples -> 112x112 (no scaling, under 672). + let (h, w) = p.vllm_resize(100, 100); + assert_eq!(h, 112); + assert_eq!(w, 112); + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + } + + #[test] + fn test_resize_exact_max_size() { + let p = MiniMaxM3Processor::new(); + // 672x672 is already factor-aligned and at the bound: unchanged. + let (h, w) = p.vllm_resize(672, 672); + assert_eq!((h, w), (672, 672)); + // Sanity: 672x672 -> grid 48x48 -> 576 tokens == config image_seq_length. + let (t, gh, gw) = p.calculate_grid_thw(h, w, 1); + assert_eq!(p.calculate_tokens_from_grid(t, gh, gw), 576); + } + + #[test] + fn test_resize_scales_down_preserving_aspect() { + let p = MiniMaxM3Processor::new(); + // 800x600 (w x h). ceil -> 812x616, exceeds max_w=672 -> scale down. + // Expected (height, width) = (532, 672); see hand-computation in the port. + let (h, w) = p.vllm_resize(600, 800); + assert_eq!(w, 672); + assert_eq!(h, 532); + assert!(w <= 672 && h <= 672); + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + } + + #[test] + fn test_calculate_num_tokens() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig::default(); + // 672x672 -> 576 tokens. + assert_eq!(p.calculate_num_tokens(672, 672, &config), 576); + // 800x600 -> grid 38x48 -> (38*48)/4 = 456. + assert_eq!(p.calculate_num_tokens(800, 600, &config), 456); + } + + #[test] + fn test_preprocess_single() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig { + do_resize: Some(true), + do_normalize: Some(true), + image_mean: Some(MINIMAX_M3_MEAN.to_vec()), + image_std: Some(MINIMAX_M3_STD.to_vec()), + ..Default::default() + }; + + let image = create_test_image(600, 400, Rgb([128, 128, 128])); + let result = p.preprocess(&[image], &config).unwrap(); + + // pixel_values is patchified: [total_patches, patch_features]. + assert_eq!(result.pixel_values.ndim(), 2); + assert_eq!(result.pixel_values.shape()[1], 3 * 2 * 14 * 14); // 1176 + assert!(result.pixel_values.shape()[0] > 0); + + assert!(result.model_specific.contains_key("image_grid_thw")); + assert!(result.model_specific.contains_key("patches_per_image")); + assert!(result.num_img_tokens[0] > 0); + } + + #[test] + fn test_preprocess_multiple() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig::default(); + + let images = vec![ + create_test_image(600, 400, Rgb([100, 100, 100])), + create_test_image(400, 600, Rgb([150, 150, 150])), + ]; + + let result = p.preprocess(&images, &config).unwrap(); + + assert_eq!(result.image_sizes.len(), 2); + assert_eq!(result.num_img_tokens.len(), 2); + assert_eq!(result.pixel_values.ndim(), 2); + + if let Some(ModelSpecificValue::IntTensor { data, shape }) = + result.model_specific.get("image_grid_thw") + { + assert_eq!(shape, &[2, 3]); + assert_eq!(data.len(), 6); + } else { + panic!("Expected image_grid_thw to be IntTensor"); + } + + if let Some(ModelSpecificValue::IntTensor { data, .. }) = + result.model_specific.get("patches_per_image") + { + let total: i64 = data.iter().sum(); + assert_eq!(total as usize, result.pixel_values.shape()[0]); + } else { + panic!("Expected patches_per_image to be IntTensor"); + } + } + + #[test] + fn test_preprocess_empty_batch_errors() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig::default(); + assert!(p.preprocess(&[], &config).is_err()); + } + + #[test] + fn test_from_preprocessor_config() { + let mut size = std::collections::HashMap::new(); + size.insert("height".to_string(), 1008u32); + size.insert("width".to_string(), 672u32); + let config = PreProcessorConfig { + merge_size: Some(2), + temporal_patch_size: Some(2), + size: Some(size), + ..Default::default() + }; + let p = MiniMaxM3Processor::from_preprocessor_config(&config); + assert_eq!(p.patch_size(), 14); + assert_eq!(p.merge_size(), 2); + // max_size is (width, height) = (672, 1008). + assert_eq!(p.max_size(), (672, 1008)); + } +} diff --git a/src/vision/processors/mod.rs b/src/vision/processors/mod.rs index eeded26..310bbcf 100644 --- a/src/vision/processors/mod.rs +++ b/src/vision/processors/mod.rs @@ -14,10 +14,12 @@ //! - **Phi4-Vision** (`phi4_vision`): Dynamic HD transform with 448x448 tiles and SiGLIP encoder //! - **LLaMA 4 Vision** (`llama4_vision`): Tile-based processing with 336x336 tiles and global tile //! - **Pixtral/Mistral3** (`pixtral`): CLIP-based preprocessing with dynamic resolution +//! - **MiniMax-M3** (`minimax_m3`): Qwen2-VL patchify with vLLM-style resize (max_size bound) pub mod kimi_k25; pub mod llama4_vision; pub mod llava; +pub mod minimax_m3; pub mod phi3_vision; pub mod phi4_vision; pub mod pixtral; @@ -28,6 +30,7 @@ pub mod qwen_vl_base; pub use kimi_k25::KimiK25Processor; pub use llama4_vision::Llama4VisionProcessor; pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor}; +pub use minimax_m3::MiniMaxM3Processor; pub use phi3_vision::Phi3VisionProcessor; pub use phi4_vision::Phi4VisionProcessor; pub use pixtral::PixtralProcessor; From 3589fb70ea619735d0d38f0ef90e25276778696a Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 4 Jun 2026 03:48:38 +0000 Subject: [PATCH 2/6] fix and update test Signed-off-by: Isotr0py --- scripts/generate_vision_golden.py | 79 ++++++++++++++ src/vision/mod.rs | 4 +- src/vision/processors/minimax_m3.rs | 119 +++++++++++++--------- tests/vision_golden_tests.rs | 153 +++++++++++++++++++++++++++- 4 files changed, 303 insertions(+), 52 deletions(-) diff --git a/scripts/generate_vision_golden.py b/scripts/generate_vision_golden.py index a3f29a6..c7a77c4 100755 --- a/scripts/generate_vision_golden.py +++ b/scripts/generate_vision_golden.py @@ -72,6 +72,11 @@ "processor_class": "PixtralImageProcessor", "description": "Dynamic resolution with CLIP normalization and bicubic resize", }, + "minimax_m3": { + "model_id": "MiniMaxAI/Minimax-M3-preview", + "processor_class": "MiniMaxM3VLImageProcessor", + "description": "Qwen2-VL patchify with vLLM-style resize (max_size bound), CLIP normalization", + }, } # Default test images @@ -607,6 +612,79 @@ def generate_golden_pixtral(image_path: str, output_dir: str) -> dict: return result +def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict: + """Generate golden output for MiniMax-M3 VL. + + MiniMax-M3 is "Qwen2VLImageProcessorFast with resize changed to vLLM style": + the patchify pipeline is identical to Qwen2-VL, but instead of smart-resize + (a min/max pixel budget) it uses ``get_hw_multiple_of``: + 1. Round each dimension up to a multiple of (patch_size * merge_size) + 2. If a dimension exceeds max_size (width, height), scale down preserving + aspect ratio, then re-align (round up) to the factor + 3. Normalize with CLIP mean/std + 4. Returns image_grid_thw for position encoding + + Default parameters: + - patch_size: 14 + - merge_size: 2 + - temporal_patch_size: 2 + - max_size: (672, 672) (inferred from size = {height: 672, width: 672}) + """ + from transformers import AutoImageProcessor + + # The image processor is registered under AutoImageProcessor in the model's + # auto_map; instantiate it directly to avoid the tokenizer dependency of the + # full AutoProcessor. Set MINIMAX_M3_MODEL_PATH to load from a local snapshot + # (e.g. for offline generation). + model_ref = os.environ.get("MINIMAX_M3_MODEL_PATH", "MiniMaxAI/Minimax-M3-preview") + img_processor = AutoImageProcessor.from_pretrained(model_ref, trust_remote_code=True) + image = Image.open(image_path).convert("RGB") + original_size = image.size + + # Process image + outputs = img_processor(images=image, return_tensors="pt") + + # Convert to numpy for saving + pixel_values = outputs["pixel_values"].numpy() + image_grid_thw = outputs.get("image_grid_thw") + if image_grid_thw is not None: + image_grid_thw = image_grid_thw.numpy() + + # Get config values + patch_size = getattr(img_processor, "patch_size", 14) + merge_size = getattr(img_processor, "merge_size", 2) + temporal_patch_size = getattr(img_processor, "temporal_patch_size", 2) + + # Calculate number of tokens: (T * H * W) / merge_size² + if image_grid_thw is not None: + grid_thw = image_grid_thw[0] + num_tokens = int(np.prod(grid_thw) / (merge_size**2)) + else: + num_tokens = None + + result = { + "pixel_values": pixel_values, + "original_size": original_size, + "processor_config": img_processor.to_dict(), + } + + if image_grid_thw is not None: + result["image_grid_thw"] = image_grid_thw + + if num_tokens is not None: + result["num_tokens"] = num_tokens + + # Add debug info + result["config_info"] = { + "patch_size": patch_size, + "merge_size": merge_size, + "temporal_patch_size": temporal_patch_size, + "max_size": getattr(img_processor, "max_size", None), + } + + return result + + def generate_for_model(model_key: str, image_paths: list, output_dir: str): """Generate golden outputs for a specific model.""" print(f"\nGenerating golden outputs for {model_key}...") @@ -621,6 +699,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str): "phi4_vision": generate_golden_phi4_vision, "llama4_vision": generate_golden_llama4_vision, "pixtral": generate_golden_pixtral, + "minimax_m3": generate_golden_minimax_m3, }.get(model_key) if generator_fn is None: diff --git a/src/vision/mod.rs b/src/vision/mod.rs index f980b1a..273195d 100644 --- a/src/vision/mod.rs +++ b/src/vision/mod.rs @@ -40,7 +40,7 @@ pub use image_processor::{ }; pub use preprocessor_config::PreProcessorConfig; pub use processors::{ - Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, Phi3VisionProcessor, - Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor, + Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, MiniMaxM3Processor, + Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor, }; pub use transforms::TransformError; diff --git a/src/vision/processors/minimax_m3.rs b/src/vision/processors/minimax_m3.rs index 597ffd2..10a826e 100644 --- a/src/vision/processors/minimax_m3.rs +++ b/src/vision/processors/minimax_m3.rs @@ -43,16 +43,21 @@ pub const DEFAULT_MERGE_SIZE: usize = 2; /// Default temporal patch size (for video frames; images repeat the single frame). pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; -/// Default vLLM-style resize bound as `(max_width, max_height)`. +/// The `size`-derived resize bound `(max_width, max_height)` from the HF source +/// (`size = {"height": 672, "width": 672}`). Must be divisible by the factor +/// (`patch_size * merge_size`); 672 / 28 = 24. /// -/// Inferred from the HF default `size = {"height": 672, "width": 672}`. Must be -/// divisible by the factor (`patch_size * merge_size`); 672 / 28 = 24. -pub const DEFAULT_MAX_SIZE: (usize, usize) = (672, 672); +/// NOTE: under current `transformers`, the model's `_further_process_kwargs` +/// hook (which would feed this into the resize) is not invoked, so the live HF +/// processor runs with `max_size = None` (no clamping). The default constructor +/// matches that observed behavior; pass `Some(MINIMAX_M3_SIZE_BOUND)` to enforce +/// the source's intended 672 clamp. +pub const MINIMAX_M3_SIZE_BOUND: (usize, usize) = (672, 672); /// Round `x` up to the nearest multiple of `multiple`. #[inline] fn ceil_to_multiple(x: usize, multiple: usize) -> usize { - if multiple == 0 || x % multiple == 0 { + if multiple == 0 || x.is_multiple_of(multiple) { x } else { x + (multiple - x % multiple) @@ -66,8 +71,12 @@ fn ceil_to_multiple(x: usize, multiple: usize) -> usize { #[derive(Debug, Clone)] pub struct MiniMaxM3Processor { inner: QwenVLProcessorBase, - /// vLLM-style resize bound as `(max_width, max_height)`. - max_size: (usize, usize), + /// Optional vLLM-style resize bound as `(max_width, max_height)`. + /// + /// `None` means no upper bound (each dimension is only rounded up to the + /// alignment factor) — this matches the live HF processor's behavior. + /// `Some((w, h))` clamps the image to fit within `(w, h)` before re-aligning. + max_size: Option<(usize, usize)>, } impl Default for MiniMaxM3Processor { @@ -83,14 +92,14 @@ impl MiniMaxM3Processor { /// - patch_size: 14 /// - merge_size: 2 /// - temporal_patch_size: 2 - /// - max_size: (672, 672) + /// - max_size: `None` (no clamp — matches the live HF processor) /// - normalization: CLIP mean/std pub fn new() -> Self { Self::with_config( DEFAULT_PATCH_SIZE, DEFAULT_MERGE_SIZE, DEFAULT_TEMPORAL_PATCH_SIZE, - DEFAULT_MAX_SIZE, + None, ) } @@ -99,7 +108,7 @@ impl MiniMaxM3Processor { patch_size: usize, merge_size: usize, temporal_patch_size: usize, - max_size: (usize, usize), + max_size: Option<(usize, usize)>, ) -> Self { Self { // min_pixels / max_pixels are unused: MiniMax never calls smart_resize. @@ -118,18 +127,16 @@ impl MiniMaxM3Processor { } /// Create a processor from a HuggingFace preprocessor config. + /// + /// `max_size` is left unset (`None`) to match the live HF processor, which + /// runs without the `size`-derived clamp under current `transformers`. pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { let patch_size = config.get_patch_size(DEFAULT_PATCH_SIZE); let merge_size = config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE); let temporal_patch_size = config .temporal_patch_size .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE); - // `get_target_size` returns (height, width); max_size is (width, height). - let max_size = config - .get_target_size() - .map(|(h, w)| (w as usize, h as usize)) - .unwrap_or(DEFAULT_MAX_SIZE); - Self::with_config(patch_size, merge_size, temporal_patch_size, max_size) + Self::with_config(patch_size, merge_size, temporal_patch_size, None) } /// Get the patch size. @@ -147,8 +154,8 @@ impl MiniMaxM3Processor { self.inner.temporal_patch_size() } - /// Get the vLLM-style resize bound as `(max_width, max_height)`. - pub fn max_size(&self) -> (usize, usize) { + /// Get the optional vLLM-style resize bound as `(max_width, max_height)`. + pub fn max_size(&self) -> Option<(usize, usize)> { self.max_size } @@ -158,29 +165,31 @@ impl MiniMaxM3Processor { self.inner.get_factor() } - /// Compute the target `(new_width, new_height)`, both multiples of `factor`, - /// scaled to fit within `max_size`. Mirrors HF `get_hw_multiple_of` for the - /// `(max_w, max_h)` tuple case (the only one MiniMax uses). + /// Compute the target `(new_width, new_height)`, both multiples of `factor`. + /// When `max_size` is set, the image is scaled down to fit within it while + /// preserving aspect ratio, then re-aligned. Mirrors HF `get_hw_multiple_of` + /// (the `(max_w, max_h)` tuple / `None` cases). fn get_hw_multiple_of(&self, width: usize, height: usize, factor: usize) -> (usize, usize) { - let (max_w, max_h) = self.max_size; let mut new_w = ceil_to_multiple(width, factor); let mut new_h = ceil_to_multiple(height, factor); - if new_w > max_w || new_h > max_h { - // Scale down to fit within max_size while maintaining aspect ratio. - // (new_w * max_w) // new_w == max_w, kept explicit to match HF. - let new_w_ = max_w.min(new_w * max_h / new_h); - let new_h_ = (new_h * max_w / new_w).min(max_h); - // Re-align (round up) to the factor. - new_w = ceil_to_multiple(new_w_, factor); - new_h = ceil_to_multiple(new_h_, factor); + if let Some((max_w, max_h)) = self.max_size { + if new_w > max_w || new_h > max_h { + // Scale down to fit within max_size while maintaining aspect ratio. + // (new_w * max_w) // new_w == max_w, kept explicit to match HF. + let new_w_ = max_w.min(new_w * max_h / new_h); + let new_h_ = (new_h * max_w / new_w).min(max_h); + // Re-align (round up) to the factor. + new_w = ceil_to_multiple(new_w_, factor); + new_h = ceil_to_multiple(new_h_, factor); + } } (new_w, new_h) } /// vLLM-style resize. Returns `(new_height, new_width)`, both multiples of the - /// alignment factor and bounded by `max_size`. + /// alignment factor and (when `max_size` is set) bounded by it. pub fn vllm_resize(&self, height: usize, width: usize) -> (usize, usize) { let factor = self.get_factor(); let (new_w, new_h) = self.get_hw_multiple_of(width, height, factor); @@ -343,7 +352,8 @@ mod tests { assert_eq!(p.merge_size(), 2); assert_eq!(p.temporal_patch_size(), 2); assert_eq!(p.get_factor(), 28); // 14 * 2 - assert_eq!(p.max_size(), (672, 672)); + // No clamp by default — matches the live HF processor (max_size=None). + assert_eq!(p.max_size(), None); } #[test] @@ -372,7 +382,6 @@ mod tests { #[test] fn test_resize_exact_max_size() { let p = MiniMaxM3Processor::new(); - // 672x672 is already factor-aligned and at the bound: unchanged. let (h, w) = p.vllm_resize(672, 672); assert_eq!((h, w), (672, 672)); // Sanity: 672x672 -> grid 48x48 -> 576 tokens == config image_seq_length. @@ -381,13 +390,30 @@ mod tests { } #[test] - fn test_resize_scales_down_preserving_aspect() { + fn test_resize_default_no_clamp() { + // Default (max_size=None) matches the live HF processor: dimensions are + // only rounded up to the factor, never scaled down. These values were + // verified against HF transformers golden output. let p = MiniMaxM3Processor::new(); + + // 500x500 -> 504x504 -> grid 36x36 -> 324 tokens. + assert_eq!(p.vllm_resize(500, 500), (504, 504)); + // 4000x3000 (w x h): ceil-aligned, NOT clamped -> grid 216x286. + let (h, w) = p.vllm_resize(3000, 4000); + assert_eq!((h, w), (3024, 4004)); + let (t, gh, gw) = p.calculate_grid_thw(h, w, 1); + assert_eq!((gh, gw), (216, 286)); + assert_eq!(p.calculate_tokens_from_grid(t, gh, gw), 15444); + } + + #[test] + fn test_resize_with_clamp_scales_down() { + // With an explicit bound, large images are scaled down (the source's + // intended `size`-derived behavior). + let p = MiniMaxM3Processor::with_config(14, 2, 2, Some((672, 672))); // 800x600 (w x h). ceil -> 812x616, exceeds max_w=672 -> scale down. - // Expected (height, width) = (532, 672); see hand-computation in the port. let (h, w) = p.vllm_resize(600, 800); - assert_eq!(w, 672); - assert_eq!(h, 532); + assert_eq!((h, w), (532, 672)); assert!(w <= 672 && h <= 672); assert_eq!(h % 28, 0); assert_eq!(w % 28, 0); @@ -395,12 +421,13 @@ mod tests { #[test] fn test_calculate_num_tokens() { - let p = MiniMaxM3Processor::new(); let config = PreProcessorConfig::default(); - // 672x672 -> 576 tokens. - assert_eq!(p.calculate_num_tokens(672, 672, &config), 576); - // 800x600 -> grid 38x48 -> (38*48)/4 = 456. - assert_eq!(p.calculate_num_tokens(800, 600, &config), 456); + // Default (no clamp): 500x500 -> 324 tokens (verified against HF). + let p = MiniMaxM3Processor::new(); + assert_eq!(p.calculate_num_tokens(500, 500, &config), 324); + // With clamp: 800x600 -> grid 38x48 -> (38*48)/4 = 456. + let pc = MiniMaxM3Processor::with_config(14, 2, 2, Some((672, 672))); + assert_eq!(pc.calculate_num_tokens(800, 600, &config), 456); } #[test] @@ -471,19 +498,15 @@ mod tests { #[test] fn test_from_preprocessor_config() { - let mut size = std::collections::HashMap::new(); - size.insert("height".to_string(), 1008u32); - size.insert("width".to_string(), 672u32); let config = PreProcessorConfig { merge_size: Some(2), temporal_patch_size: Some(2), - size: Some(size), ..Default::default() }; let p = MiniMaxM3Processor::from_preprocessor_config(&config); assert_eq!(p.patch_size(), 14); assert_eq!(p.merge_size(), 2); - // max_size is (width, height) = (672, 1008). - assert_eq!(p.max_size(), (672, 1008)); + // No clamp by default — matches the live HF processor. + assert_eq!(p.max_size(), None); } } diff --git a/tests/vision_golden_tests.rs b/tests/vision_golden_tests.rs index cd6e500..caf8649 100644 --- a/tests/vision_golden_tests.rs +++ b/tests/vision_golden_tests.rs @@ -8,6 +8,7 @@ //! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad) //! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models) //! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models) +//! - `minimax_m3/` - Qwen2-VL patchify with vLLM-style resize (MiniMaxAI/Minimax-M3-* models) //! //! To regenerate golden outputs: //! ```bash @@ -32,8 +33,8 @@ use std::{fs::File, io::Read, path::Path}; use llm_multimodal::vision::{ image_processor::ModelSpecificValue, ImagePreProcessor, Llama4VisionProcessor, LlavaProcessor, - Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, PreProcessorConfig, - Qwen2VLProcessor, Qwen3VLProcessor, + MiniMaxM3Processor, Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, + PreProcessorConfig, Qwen2VLProcessor, Qwen3VLProcessor, }; use ndarray::{Array4, Array5}; @@ -611,6 +612,154 @@ fn test_qwen3_vl_golden_grayscale() { run_qwen3_vl_golden_test("grayscale"); } +// ============================================================================ +// MiniMax-M3 tests +// ============================================================================ + +/// Run a MiniMax-M3 golden test for a specific image. +/// +/// This test validates: +/// 1. image_grid_thw matches the HuggingFace output +/// 2. num_tokens calculation is correct +/// 3. Pixel values match after patchification +/// +/// MiniMax-M3 shares Qwen2-VL's patchify pipeline (patch_size=14, merge_size=2, +/// CLIP normalization) but uses vLLM-style resize bounded by `max_size` instead +/// of Qwen's min/max-pixel smart resize. +fn run_minimax_m3_golden_test(image_name: &str) { + let golden_dir = Path::new("crates/multimodal/tests/fixtures/golden/minimax_m3"); + let image_path = + Path::new("crates/multimodal/tests/fixtures/images").join(format!("{image_name}.jpg")); + + if !golden_dir.exists() || !image_path.exists() { + eprintln!("Golden test fixtures for minimax_m3/{image_name} not found, skipping test"); + eprintln!( + "Run: python crates/multimodal/scripts/generate_vision_golden.py --model minimax_m3" + ); + return; + } + + let npz_path = golden_dir.join(format!("golden_{image_name}.npz")); + let config = load_config(&golden_dir.join("preprocessor_config.json")); + + // Load golden values + let golden_grid_thw = load_golden_grid_thw(&npz_path); + let golden_num_tokens = load_golden_num_tokens(&npz_path); + let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path); + + // Process image with our Rust processor + let image = image::open(&image_path).expect("Failed to open image"); + let processor = MiniMaxM3Processor::from_preprocessor_config(&config); + let result = processor + .preprocess(&[image], &config) + .expect("Processing failed"); + + // Extract image_grid_thw from result + let rust_grid_thw = match result.model_specific.get("image_grid_thw") { + Some(ModelSpecificValue::IntTensor { data, shape }) => { + assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image"); + data.clone() + } + _ => panic!("Expected image_grid_thw in model_specific"), + }; + + // Compare grid dimensions + println!( + "minimax_m3 - {image_name} image - Grid T H W: golden={golden_grid_thw:?}, rust={rust_grid_thw:?}" + ); + assert_eq!( + golden_grid_thw, rust_grid_thw, + "image_grid_thw mismatch for {image_name}" + ); + + // Compare token counts + let rust_num_tokens = result.num_img_tokens[0]; + println!( + "minimax_m3 - {image_name} image - Tokens: golden={golden_num_tokens}, rust={rust_num_tokens}" + ); + assert_eq!( + golden_num_tokens, rust_num_tokens, + "num_tokens mismatch for {image_name}" + ); + + // pixel_values is already patchified: [total_patches, patch_features] + let rust_patches = result.pixel_values_flat(); + let rust_shape = ( + result.pixel_values.shape()[0], + result.pixel_values.shape()[1], + ); + + println!( + "minimax_m3 - {image_name} image - Patch shape: golden={golden_shape:?}, rust={rust_shape:?}" + ); + assert_eq!(golden_shape, rust_shape, "Patch shape mismatch"); + + // Compare pixel values + let max_diff = rust_patches + .iter() + .zip(golden_pixels.iter()) + .map(|(r, g)| (r - g).abs()) + .fold(0.0f32, f32::max); + + println!("minimax_m3 - {image_name} image - Max pixel diff: {max_diff:.6}"); + + // Allow tolerance for floating point and interpolation differences + assert!( + max_diff < 0.1, + "Max pixel difference {max_diff} exceeds tolerance 0.1 for {image_name}" + ); +} + +#[test] +fn test_minimax_m3_golden_square() { + run_minimax_m3_golden_test("square"); +} + +#[test] +fn test_minimax_m3_golden_tall() { + run_minimax_m3_golden_test("tall"); +} + +#[test] +fn test_minimax_m3_golden_wide() { + run_minimax_m3_golden_test("wide"); +} + +#[test] +fn test_minimax_m3_golden_small() { + run_minimax_m3_golden_test("small"); +} + +#[test] +fn test_minimax_m3_golden_tiny() { + run_minimax_m3_golden_test("tiny"); +} + +#[test] +fn test_minimax_m3_golden_very_tall() { + run_minimax_m3_golden_test("very_tall"); +} + +#[test] +fn test_minimax_m3_golden_very_wide() { + run_minimax_m3_golden_test("very_wide"); +} + +#[test] +fn test_minimax_m3_golden_large() { + run_minimax_m3_golden_test("large"); +} + +#[test] +fn test_minimax_m3_golden_odd_dims() { + run_minimax_m3_golden_test("odd_dims"); +} + +#[test] +fn test_minimax_m3_golden_grayscale() { + run_minimax_m3_golden_test("grayscale"); +} + // ============================================================================ // Phi3-Vision tests // ============================================================================ From 1f5da2675394a1ab237bf2834d6ad86e3bc2b84d Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 4 Jun 2026 06:33:27 +0000 Subject: [PATCH 3/6] add processor spec Signed-off-by: Isotr0py --- src/registry/minimax_m3.rs | 168 ++++++++++++++++++++++++++++++ src/registry/mod.rs | 3 + src/vision/preprocessor_config.rs | 77 +++++++++++++- 3 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 src/registry/minimax_m3.rs diff --git a/src/registry/minimax_m3.rs b/src/registry/minimax_m3.rs new file mode 100644 index 0000000..2350d76 --- /dev/null +++ b/src/registry/minimax_m3.rs @@ -0,0 +1,168 @@ +use std::collections::HashMap; + +use serde_json::{json, Value}; + +use crate::{ + registry::{ModelMetadata, ModelProcessorSpec, RegistryResult}, + types::{FieldLayout, Modality, PromptReplacement, TokenId}, + vision::image_processor::PreprocessedImages, +}; + +pub(super) struct MiniMaxM3VisionSpec; + +impl MiniMaxM3VisionSpec { + const IMAGE_TOKEN: &'static str = "]<]image[>["; + const VISION_START_TOKEN: &'static str = "]<]start of image[>["; + const VISION_END_TOKEN: &'static str = "]<]end of image[>["; +} + +impl ModelProcessorSpec for MiniMaxM3VisionSpec { + fn name(&self) -> &'static str { + "minimax_m3_vl" + } + + fn matches(&self, metadata: &ModelMetadata) -> bool { + let id = metadata.model_id.to_ascii_lowercase(); + id.contains("minimax") && id.contains("m3") + || metadata + .config_model_type() + .is_some_and(|mt| mt == "minimax_m3_vl") + } + + fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult { + Ok(Self::IMAGE_TOKEN.to_string()) + } + + fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult { + metadata.token_id(Self::IMAGE_TOKEN) + } + + fn modality_limits( + &self, + _metadata: &ModelMetadata, + ) -> RegistryResult> { + Ok(HashMap::from([(Modality::Image, 64)])) + } + + fn processor_kwargs(&self, _metadata: &ModelMetadata) -> RegistryResult { + Ok(json!({})) + } + + fn prompt_replacements( + &self, + metadata: &ModelMetadata, + preprocessed: &PreprocessedImages, + ) -> RegistryResult> { + let image_token_id = metadata.token_id(Self::IMAGE_TOKEN)?; + let start_token_id = metadata.token_id(Self::VISION_START_TOKEN)?; + let end_token_id = metadata.token_id(Self::VISION_END_TOKEN)?; + + Ok(preprocessed + .num_img_tokens + .iter() + .map(|&n| { + // Mirrors MiniMaxM3VLMultiModalProcessor._get_prompt_updates: + // full = [start_token_id] + [image_token_id] * N + [end_token_id] + // The image_token_id positions are marked as embed tokens. + let mut tokens = Vec::with_capacity(n + 2); + tokens.push(start_token_id); + tokens.extend(std::iter::repeat_n(image_token_id, n)); + tokens.push(end_token_id); + PromptReplacement::sequence(Modality::Image, Self::IMAGE_TOKEN, tokens) + }) + .collect()) + } + + fn field_layouts(&self) -> HashMap { + HashMap::from([ + ( + "pixel_values".to_string(), + FieldLayout::flat("patches_per_image"), + ), + ("image_grid_thw".to_string(), FieldLayout::Batched), + ("patches_per_image".to_string(), FieldLayout::Batched), + ]) + } + + fn keep_on_cpu_keys(&self) -> Vec { + vec!["image_grid_thw".to_string()] + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::{ + registry::{test_helpers::*, ModelMetadata, ModelRegistry}, + types::ImageSize, + }; + + #[test] + fn minimax_m3_matches_model_type() { + let tokenizer = TestTokenizer::new(&[ + ("]<]image[>[", 200025), + ("]<]start of image[>[", 200029), + ("]<]end of image[>[", 200030), + ]); + let config = json!({ "model_type": "minimax_m3_vl" }); + let metadata = ModelMetadata { + model_id: "some-custom-model", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec"); + assert_eq!(spec.name(), "minimax_m3_vl"); + } + + #[test] + fn minimax_m3_matches_model_id() { + let tokenizer = TestTokenizer::new(&[ + ("]<]image[>[", 200025), + ("]<]start of image[>[", 200029), + ("]<]end of image[>[", 200030), + ]); + let config = json!({ "model_type": "minimax_m3_vl" }); + let metadata = ModelMetadata { + model_id: "MiniMaxAI/Minimax-M3-preview", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec"); + assert_eq!(spec.name(), "minimax_m3_vl"); + } + + #[test] + fn minimax_m3_prompt_replacements_wrap_with_start_end() { + let tokenizer = TestTokenizer::new(&[ + ("]<]image[>[", 200025), + ("]<]start of image[>[", 200029), + ("]<]end of image[>[", 200030), + ]); + let config = json!({ "model_type": "minimax_m3_vl" }); + let metadata = ModelMetadata { + model_id: "MiniMaxAI/Minimax-M3-preview", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec"); + + // 672x672 image → 576 tokens (48x48 grid, merge_size=2 → 48*48/4=576) + let replacements = spec + .prompt_replacements( + &metadata, + &test_preprocessed_with_tokens(&[ImageSize::new(672, 672)], &[576]), + ) + .unwrap(); + + assert_eq!(replacements.len(), 1); + let tokens = &replacements[0].tokens; + assert_eq!(tokens.len(), 578); // 1 start + 576 image + 1 end + assert_eq!(tokens[0], 200029); // start + assert!(tokens[1..577].iter().all(|&t| t == 200025)); // image tokens + assert_eq!(tokens[577], 200030); // end + } +} diff --git a/src/registry/mod.rs b/src/registry/mod.rs index e0cbc28..0554b65 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -1,6 +1,7 @@ mod kimi_k25; mod llama4; mod llava; +mod minimax_m3; mod phi3_v; mod qwen3_vl; mod qwen_vl; @@ -9,6 +10,7 @@ mod traits; use kimi_k25::KimiK25VisionSpec; use llama4::Llama4Spec; use llava::{LlavaNextSpec, LlavaSpec}; +use minimax_m3::MiniMaxM3VisionSpec; use once_cell::sync::Lazy; use phi3_v::Phi3VisionSpec; use qwen3_vl::Qwen3VLVisionSpec; @@ -31,6 +33,7 @@ impl ModelRegistry { // LlavaNext must be registered before Llava so "llava_next" model_type matches first. LazySpec::new("llava_next", || Box::new(LlavaNextSpec)), LazySpec::new("llava", || Box::new(LlavaSpec)), + LazySpec::new("minimax_m3_vl", || Box::new(MiniMaxM3VisionSpec)), // Qwen3-VL must be registered before QwenVL so "qwen3" matches first. LazySpec::new("qwen3_vl", || Box::new(Qwen3VLVisionSpec)), LazySpec::new("qwen_vl", || Box::new(QwenVLVisionSpec)), diff --git a/src/vision/preprocessor_config.rs b/src/vision/preprocessor_config.rs index d03251d..da166a9 100644 --- a/src/vision/preprocessor_config.rs +++ b/src/vision/preprocessor_config.rs @@ -17,6 +17,72 @@ pub struct PatchSize { pub width: Option, } +/// Custom deserializer for the `size` field that handles both map and array formats. +/// - Map format: `"size": {"height": 672, "width": 672}` (standard HuggingFace) +/// - Array format: `"size": [672, 672]` -> treated as [height, width] (MiniMax M3) +fn deserialize_size<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + use std::fmt; + + use serde::de::{self, MapAccess, SeqAccess, Visitor}; + + struct SizeVisitor; + + impl<'de> Visitor<'de> for SizeVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a map with height/width/shortest_edge, an array [height, width], or null") + } + + fn visit_none(self) -> Result + where + E: de::Error, + { + Ok(None) + } + + fn visit_unit(self) -> Result + where + E: de::Error, + { + Ok(None) + } + + fn visit_map(self, mut map: M) -> Result + where + M: MapAccess<'de>, + { + let mut result = HashMap::new(); + while let Some(key) = map.next_key::()? { + let value = map.next_value::()?; + result.insert(key, value); + } + Ok(Some(result)) + } + + fn visit_seq(self, mut seq: S) -> Result + where + S: SeqAccess<'de>, + { + let h: u32 = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &"array of [height, width]"))?; + let w: u32 = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &"array of [height, width]"))?; + let mut result = HashMap::new(); + result.insert("height".to_string(), h); + result.insert("width".to_string(), w); + Ok(Some(result)) + } + } + + deserializer.deserialize_any(SizeVisitor) +} + /// Custom deserializer for patch_size that handles both integer and dict formats. /// - Integer format: `"patch_size": 16` -> PatchSize { height: 16, width: 16 } /// - Dict format: `"patch_size": {"height": 16, "width": 16}` -> PatchSize { height: 16, width: 16 } @@ -148,12 +214,12 @@ pub struct PreProcessorConfig { pub resampling: Option, /// Target size for resizing - /// Can be {"height": H, "width": W} or {"shortest_edge": S} - #[serde(default)] + /// Can be {"height": H, "width": W}, {"shortest_edge": S}, or [H, W] + #[serde(default, deserialize_with = "deserialize_size")] pub size: Option>, /// Target size for center cropping - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_size")] pub crop_size: Option>, // ===================== @@ -475,6 +541,11 @@ mod tests { let json2 = r#"{"size": {"shortest_edge": 224}}"#; let config2 = PreProcessorConfig::from_json(json2).unwrap(); assert_eq!(config2.get_target_size(), Some((224, 224))); + + // Array format [height, width] (e.g. MiniMax M3) + let json3 = r#"{"size": [672, 672]}"#; + let config3 = PreProcessorConfig::from_json(json3).unwrap(); + assert_eq!(config3.get_target_size(), Some((672, 672))); } #[test] From 52bb5369e75537aea438f8c2c9a7385034fe2bb3 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 4 Jun 2026 06:48:18 +0000 Subject: [PATCH 4/6] clean Signed-off-by: Isotr0py --- scripts/generate_vision_golden.py | 3 +-- src/vision/image_processor.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/generate_vision_golden.py b/scripts/generate_vision_golden.py index c7a77c4..0ea6cf5 100755 --- a/scripts/generate_vision_golden.py +++ b/scripts/generate_vision_golden.py @@ -636,8 +636,7 @@ def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict: # auto_map; instantiate it directly to avoid the tokenizer dependency of the # full AutoProcessor. Set MINIMAX_M3_MODEL_PATH to load from a local snapshot # (e.g. for offline generation). - model_ref = os.environ.get("MINIMAX_M3_MODEL_PATH", "MiniMaxAI/Minimax-M3-preview") - img_processor = AutoImageProcessor.from_pretrained(model_ref, trust_remote_code=True) + img_processor = AutoImageProcessor.from_pretrained("MiniMaxAI/Minimax-M3-preview", trust_remote_code=True) image = Image.open(image_path).convert("RGB") original_size = image.size diff --git a/src/vision/image_processor.rs b/src/vision/image_processor.rs index cf3272f..c4a0c38 100644 --- a/src/vision/image_processor.rs +++ b/src/vision/image_processor.rs @@ -476,11 +476,11 @@ impl ImageProcessorRegistry { // Register MiniMax-M3 VL registry.register( - "minimax-m3", + "minimax-m3-vl", Box::new(super::processors::MiniMaxM3Processor::new()), ); registry.register( - "minimax_m3", + "minimax_m3_vl", Box::new(super::processors::MiniMaxM3Processor::new()), ); From c440287b77aa9d8fb685b9d6780da76c7877eb87 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 4 Jun 2026 16:38:51 +0800 Subject: [PATCH 5/6] simplify size deserialization Signed-off-by: Bugen Zhao --- Cargo.toml | 1 + src/vision/preprocessor_config.rs | 97 ++++++++++--------------------- 2 files changed, 32 insertions(+), 66 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cae4956..b6572a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ once_cell = "1.21.3" reqwest = { version = "0.12.8", default-features = false, features = ["stream", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["preserve_order"] } +serde_with = "3.12" thiserror = "2.0.12" tokio = { version = "1.42.0", features = ["sync", "fs", "rt-multi-thread"] } url = "2.5.4" diff --git a/src/vision/preprocessor_config.rs b/src/vision/preprocessor_config.rs index da166a9..560fd71 100644 --- a/src/vision/preprocessor_config.rs +++ b/src/vision/preprocessor_config.rs @@ -7,80 +7,42 @@ use std::collections::HashMap; use image::imageops::FilterType; use serde::{Deserialize, Deserializer}; +use serde_with::{serde_as, TryFromInto}; use super::transforms; -/// Struct to represent patch_size as dict {"height": x, "width": y} -#[derive(Debug, Clone, Deserialize, Default)] -pub struct PatchSize { - pub height: Option, - pub width: Option, -} - -/// Custom deserializer for the `size` field that handles both map and array formats. +/// Deserialization shape for HF size fields. +/// /// - Map format: `"size": {"height": 672, "width": 672}` (standard HuggingFace) /// - Array format: `"size": [672, 672]` -> treated as [height, width] (MiniMax M3) -fn deserialize_size<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - use std::fmt; - - use serde::de::{self, MapAccess, SeqAccess, Visitor}; - - struct SizeVisitor; - - impl<'de> Visitor<'de> for SizeVisitor { - type Value = Option>; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a map with height/width/shortest_edge, an array [height, width], or null") - } - - fn visit_none(self) -> Result - where - E: de::Error, - { - Ok(None) - } - - fn visit_unit(self) -> Result - where - E: de::Error, - { - Ok(None) - } +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum SizeRepr { + Map(HashMap), + Array([u32; 2]), +} - fn visit_map(self, mut map: M) -> Result - where - M: MapAccess<'de>, - { - let mut result = HashMap::new(); - while let Some(key) = map.next_key::()? { - let value = map.next_value::()?; - result.insert(key, value); +impl TryFrom for HashMap { + type Error = &'static str; + + fn try_from(value: SizeRepr) -> Result { + match value { + SizeRepr::Map(size) => Ok(size), + SizeRepr::Array([height, width]) => { + let mut size = HashMap::new(); + size.insert("height".to_string(), height); + size.insert("width".to_string(), width); + Ok(size) } - Ok(Some(result)) - } - - fn visit_seq(self, mut seq: S) -> Result - where - S: SeqAccess<'de>, - { - let h: u32 = seq - .next_element()? - .ok_or_else(|| de::Error::invalid_length(0, &"array of [height, width]"))?; - let w: u32 = seq - .next_element()? - .ok_or_else(|| de::Error::invalid_length(1, &"array of [height, width]"))?; - let mut result = HashMap::new(); - result.insert("height".to_string(), h); - result.insert("width".to_string(), w); - Ok(Some(result)) } } +} - deserializer.deserialize_any(SizeVisitor) +/// Struct to represent patch_size as dict {"height": x, "width": y} +#[derive(Debug, Clone, Deserialize, Default)] +pub struct PatchSize { + pub height: Option, + pub width: Option, } /// Custom deserializer for patch_size that handles both integer and dict formats. @@ -167,6 +129,7 @@ where /// /// This struct captures the common fields across different vision model processors. /// Model-specific fields are accessed via the flexible `extra` field. +#[serde_as] #[derive(Debug, Clone, Deserialize, Default)] pub struct PreProcessorConfig { /// Processor class name (e.g., "CLIPImageProcessor", "Qwen2VLImageProcessor") @@ -215,11 +178,13 @@ pub struct PreProcessorConfig { /// Target size for resizing /// Can be {"height": H, "width": W}, {"shortest_edge": S}, or [H, W] - #[serde(default, deserialize_with = "deserialize_size")] + #[serde_as(deserialize_as = "Option>")] + #[serde(default)] pub size: Option>, /// Target size for center cropping - #[serde(default, deserialize_with = "deserialize_size")] + #[serde_as(deserialize_as = "Option>")] + #[serde(default)] pub crop_size: Option>, // ===================== From 932392fdc55f8006fe5fdc80340b6ecb771aa420 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 26 Jun 2026 09:04:12 +0000 Subject: [PATCH 6/6] follow release version behavior & adapt golden tests Signed-off-by: Bugen Zhao --- .github/workflows/golden-fixtures.yml | 3 + scripts/generate_vision_golden.py | 27 ++-- src/registry/minimax_m3.rs | 4 +- src/vision/processors/minimax_m3.rs | 194 +++++++++----------------- src/vision/processors/mod.rs | 2 +- tests/vision_golden_tests.rs | 16 +-- 6 files changed, 95 insertions(+), 151 deletions(-) diff --git a/.github/workflows/golden-fixtures.yml b/.github/workflows/golden-fixtures.yml index 646701f..7b2719f 100644 --- a/.github/workflows/golden-fixtures.yml +++ b/.github/workflows/golden-fixtures.yml @@ -46,6 +46,9 @@ jobs: - model: pixtral test_filter: test_pixtral_golden extra_filter: "" + - model: minimax_m3 + test_filter: test_minimax_m3_golden + extra_filter: "" steps: - uses: actions/checkout@v6 - uses: actions-rust-lang/setup-rust-toolchain@v1 diff --git a/scripts/generate_vision_golden.py b/scripts/generate_vision_golden.py index 1d44d17..5ed7e8d 100755 --- a/scripts/generate_vision_golden.py +++ b/scripts/generate_vision_golden.py @@ -73,9 +73,9 @@ "description": "Dynamic resolution with CLIP normalization and bicubic resize", }, "minimax_m3": { - "model_id": "MiniMaxAI/Minimax-M3-preview", + "model_id": "MiniMaxAI/MiniMax-M3", "processor_class": "MiniMaxM3VLImageProcessor", - "description": "Qwen2-VL patchify with vLLM-style resize (max_size bound), CLIP normalization", + "description": "Qwen2-VL patchify with MiniMax smart resize, CLIP normalization", }, } @@ -615,12 +615,10 @@ def generate_golden_pixtral(image_path: str, output_dir: str) -> dict: def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict: """Generate golden output for MiniMax-M3 VL. - MiniMax-M3 is "Qwen2VLImageProcessorFast with resize changed to vLLM style": - the patchify pipeline is identical to Qwen2-VL, but instead of smart-resize - (a min/max pixel budget) it uses ``get_hw_multiple_of``: - 1. Round each dimension up to a multiple of (patch_size * merge_size) - 2. If a dimension exceeds max_size (width, height), scale down preserving - aspect ratio, then re-align (round up) to the factor + The patchify pipeline is identical to Qwen2-VL, with MiniMax's smart resize + defaults: + 1. Smart resize to fit within min/max pixel bounds + 2. Align dimensions to (patch_size * merge_size) boundary 3. Normalize with CLIP mean/std 4. Returns image_grid_thw for position encoding @@ -628,15 +626,17 @@ def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict: - patch_size: 14 - merge_size: 2 - temporal_patch_size: 2 - - max_size: (672, 672) (inferred from size = {height: 672, width: 672}) + - min_pixels: 4 * 28 * 28 = 3,136 + - max_pixels: 672 * 672 = 451,584 """ from transformers import AutoImageProcessor # The image processor is registered under AutoImageProcessor in the model's # auto_map; instantiate it directly to avoid the tokenizer dependency of the - # full AutoProcessor. Set MINIMAX_M3_MODEL_PATH to load from a local snapshot - # (e.g. for offline generation). - img_processor = AutoImageProcessor.from_pretrained("MiniMaxAI/Minimax-M3-preview", trust_remote_code=True) + # full AutoProcessor. + img_processor = AutoImageProcessor.from_pretrained( + "MiniMaxAI/MiniMax-M3", trust_remote_code=True + ) image = Image.open(image_path).convert("RGB") original_size = image.size @@ -678,7 +678,8 @@ def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict: "patch_size": patch_size, "merge_size": merge_size, "temporal_patch_size": temporal_patch_size, - "max_size": getattr(img_processor, "max_size", None), + "min_pixels": getattr(img_processor, "min_pixels", None), + "max_pixels": getattr(img_processor, "max_pixels", None), } return result diff --git a/src/registry/minimax_m3.rs b/src/registry/minimax_m3.rs index 2350d76..c3722e4 100644 --- a/src/registry/minimax_m3.rs +++ b/src/registry/minimax_m3.rs @@ -125,7 +125,7 @@ mod tests { ]); let config = json!({ "model_type": "minimax_m3_vl" }); let metadata = ModelMetadata { - model_id: "MiniMaxAI/Minimax-M3-preview", + model_id: "MiniMaxAI/MiniMax-M3", tokenizer: &tokenizer, config: &config, }; @@ -143,7 +143,7 @@ mod tests { ]); let config = json!({ "model_type": "minimax_m3_vl" }); let metadata = ModelMetadata { - model_id: "MiniMaxAI/Minimax-M3-preview", + model_id: "MiniMaxAI/MiniMax-M3", tokenizer: &tokenizer, config: &config, }; diff --git a/src/vision/processors/minimax_m3.rs b/src/vision/processors/minimax_m3.rs index 10a826e..9b0c3c0 100644 --- a/src/vision/processors/minimax_m3.rs +++ b/src/vision/processors/minimax_m3.rs @@ -1,23 +1,10 @@ //! MiniMax-M3 VL image processor. //! -//! Ported from HuggingFace `MiniMaxM3VLImageProcessor`. The model documents this -//! as "Copied from Qwen2VLImageProcessorFast with resize changed to vLLM style": -//! the patchify pipeline (rescale → normalize → reshape into -//! `[grid_t, grid_h, grid_w, ...]` patches) is identical to Qwen2-VL, so we reuse -//! [`QwenVLProcessorBase`] for it. The only difference is the resize step. +//! Ported from HuggingFace `MiniMaxM3VLImageProcessor`. The patchify pipeline +//! (rescale → normalize → reshape into `[grid_t, grid_h, grid_w, ...]` patches) +//! is identical to Qwen2-VL, so we reuse [`QwenVLProcessorBase`] for it. //! -//! # vLLM-style resize (`get_hw_multiple_of`) -//! -//! Unlike Qwen's smart-resize (which targets a min/max *pixel* budget), MiniMax: -//! -//! 1. Rounds each dimension **up** to a multiple of `patch_size * merge_size`. -//! 2. If either dimension exceeds `max_size` (width, height), scales the image -//! down to fit while preserving aspect ratio, then re-aligns (rounds up) to -//! the factor. -//! -//! There is no lower (min-pixels) bound. `max_size` is inferred from the -//! processor's `size` (default `{height: 672, width: 672}`) and must itself be -//! divisible by the factor. +//! MiniMax-M3 uses Qwen-style smart resize with `max_pixels = 672 * 672`. use image::{imageops::FilterType, DynamicImage, GenericImageView}; @@ -43,40 +30,19 @@ pub const DEFAULT_MERGE_SIZE: usize = 2; /// Default temporal patch size (for video frames; images repeat the single frame). pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; -/// The `size`-derived resize bound `(max_width, max_height)` from the HF source -/// (`size = {"height": 672, "width": 672}`). Must be divisible by the factor -/// (`patch_size * merge_size`); 672 / 28 = 24. -/// -/// NOTE: under current `transformers`, the model's `_further_process_kwargs` -/// hook (which would feed this into the resize) is not invoked, so the live HF -/// processor runs with `max_size = None` (no clamping). The default constructor -/// matches that observed behavior; pass `Some(MINIMAX_M3_SIZE_BOUND)` to enforce -/// the source's intended 672 clamp. -pub const MINIMAX_M3_SIZE_BOUND: (usize, usize) = (672, 672); - -/// Round `x` up to the nearest multiple of `multiple`. -#[inline] -fn ceil_to_multiple(x: usize, multiple: usize) -> usize { - if multiple == 0 || x.is_multiple_of(multiple) { - x - } else { - x + (multiple - x % multiple) - } -} +/// Default minimum pixels (4 * 28 * 28 = 3,136). +pub const DEFAULT_MIN_PIXELS: usize = 4 * 28 * 28; + +/// Default maximum pixels (672 * 672 = 451,584). +pub const DEFAULT_MAX_PIXELS: usize = 672 * 672; /// MiniMax-M3 VL image processor. /// -/// Wraps [`QwenVLProcessorBase`] for the shared patchify/grid logic and overrides -/// the resize with the vLLM-style [`Self::vllm_resize`]. +/// Wraps [`QwenVLProcessorBase`] for the shared smart-resize, patchify, and grid +/// logic. #[derive(Debug, Clone)] pub struct MiniMaxM3Processor { inner: QwenVLProcessorBase, - /// Optional vLLM-style resize bound as `(max_width, max_height)`. - /// - /// `None` means no upper bound (each dimension is only rounded up to the - /// alignment factor) — this matches the live HF processor's behavior. - /// `Some((w, h))` clamps the image to fit within `(w, h)` before re-aligning. - max_size: Option<(usize, usize)>, } impl Default for MiniMaxM3Processor { @@ -92,14 +58,16 @@ impl MiniMaxM3Processor { /// - patch_size: 14 /// - merge_size: 2 /// - temporal_patch_size: 2 - /// - max_size: `None` (no clamp — matches the live HF processor) + /// - min_pixels: 3,136 + /// - max_pixels: 451,584 /// - normalization: CLIP mean/std pub fn new() -> Self { Self::with_config( DEFAULT_PATCH_SIZE, DEFAULT_MERGE_SIZE, DEFAULT_TEMPORAL_PATCH_SIZE, - None, + DEFAULT_MIN_PIXELS, + DEFAULT_MAX_PIXELS, ) } @@ -108,35 +76,39 @@ impl MiniMaxM3Processor { patch_size: usize, merge_size: usize, temporal_patch_size: usize, - max_size: Option<(usize, usize)>, + min_pixels: usize, + max_pixels: usize, ) -> Self { Self { - // min_pixels / max_pixels are unused: MiniMax never calls smart_resize. inner: QwenVLProcessorBase::new(QwenVLConfig { patch_size, merge_size, - min_pixels: 0, - max_pixels: usize::MAX, + min_pixels, + max_pixels, temporal_patch_size, mean: MINIMAX_M3_MEAN, std: MINIMAX_M3_STD, model_name: "minimax-m3", }), - max_size, } } /// Create a processor from a HuggingFace preprocessor config. - /// - /// `max_size` is left unset (`None`) to match the live HF processor, which - /// runs without the `size`-derived clamp under current `transformers`. pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { let patch_size = config.get_patch_size(DEFAULT_PATCH_SIZE); let merge_size = config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE); let temporal_patch_size = config .temporal_patch_size .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE); - Self::with_config(patch_size, merge_size, temporal_patch_size, None) + let min_pixels = config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS); + let max_pixels = config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS); + Self::with_config( + patch_size, + merge_size, + temporal_patch_size, + min_pixels, + max_pixels, + ) } /// Get the patch size. @@ -154,9 +126,14 @@ impl MiniMaxM3Processor { self.inner.temporal_patch_size() } - /// Get the optional vLLM-style resize bound as `(max_width, max_height)`. - pub fn max_size(&self) -> Option<(usize, usize)> { - self.max_size + /// Get the minimum pixels. + pub fn min_pixels(&self) -> usize { + self.inner.min_pixels() + } + + /// Get the maximum pixels. + pub fn max_pixels(&self) -> usize { + self.inner.max_pixels() } /// Get the factor for dimension alignment (`patch_size * merge_size`). @@ -165,35 +142,13 @@ impl MiniMaxM3Processor { self.inner.get_factor() } - /// Compute the target `(new_width, new_height)`, both multiples of `factor`. - /// When `max_size` is set, the image is scaled down to fit within it while - /// preserving aspect ratio, then re-aligned. Mirrors HF `get_hw_multiple_of` - /// (the `(max_w, max_h)` tuple / `None` cases). - fn get_hw_multiple_of(&self, width: usize, height: usize, factor: usize) -> (usize, usize) { - let mut new_w = ceil_to_multiple(width, factor); - let mut new_h = ceil_to_multiple(height, factor); - - if let Some((max_w, max_h)) = self.max_size { - if new_w > max_w || new_h > max_h { - // Scale down to fit within max_size while maintaining aspect ratio. - // (new_w * max_w) // new_w == max_w, kept explicit to match HF. - let new_w_ = max_w.min(new_w * max_h / new_h); - let new_h_ = (new_h * max_w / new_w).min(max_h); - // Re-align (round up) to the factor. - new_w = ceil_to_multiple(new_w_, factor); - new_h = ceil_to_multiple(new_h_, factor); - } - } - - (new_w, new_h) - } - - /// vLLM-style resize. Returns `(new_height, new_width)`, both multiples of the - /// alignment factor and (when `max_size` is set) bounded by it. - pub fn vllm_resize(&self, height: usize, width: usize) -> (usize, usize) { - let factor = self.get_factor(); - let (new_w, new_h) = self.get_hw_multiple_of(width, height, factor); - (new_h, new_w) + /// Smart resize. Returns `(new_height, new_width)`. + pub fn smart_resize( + &self, + height: usize, + width: usize, + ) -> Result<(usize, usize), TransformError> { + self.inner.smart_resize(height, width) } /// Calculate the grid dimensions `(grid_t, grid_h, grid_w)` for an image. @@ -257,7 +212,7 @@ impl ImagePreProcessor for MiniMaxM3Processor { for image in images { let (w, h) = image.dimensions(); - let (target_h, target_w) = self.vllm_resize(h as usize, w as usize); + let (target_h, target_w) = self.smart_resize(h as usize, w as usize)?; // Resize to the image's own target size (skip if dimensions match). let (tw32, th32) = (target_w as u32, target_h as u32); @@ -299,8 +254,8 @@ impl ImagePreProcessor for MiniMaxM3Processor { ndarray::Array2::from_shape_vec((total_patches, patch_features), all_patches).map_err( |e| { TransformError::ShapeError(format!( - "Failed to create patchified pixel_values [{total_patches}, {patch_features}]: {e}" - )) + "Failed to create patchified pixel_values [{total_patches}, {patch_features}]: {e}" + )) }, )?; @@ -319,7 +274,12 @@ impl ImagePreProcessor for MiniMaxM3Processor { } fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { - let (new_height, new_width) = self.vllm_resize(height as usize, width as usize); + let (new_height, new_width) = self + .smart_resize(height as usize, width as usize) + .unwrap_or_else(|_| { + let factor = self.get_factor(); + (factor, factor) + }); let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1); self.calculate_tokens_from_grid(grid_t, grid_h, grid_w) } @@ -352,8 +312,8 @@ mod tests { assert_eq!(p.merge_size(), 2); assert_eq!(p.temporal_patch_size(), 2); assert_eq!(p.get_factor(), 28); // 14 * 2 - // No clamp by default — matches the live HF processor (max_size=None). - assert_eq!(p.max_size(), None); + assert_eq!(p.min_pixels(), DEFAULT_MIN_PIXELS); + assert_eq!(p.max_pixels(), DEFAULT_MAX_PIXELS); } #[test] @@ -371,8 +331,8 @@ mod tests { #[test] fn test_resize_within_bounds_aligns_up() { let p = MiniMaxM3Processor::new(); - // 100x100 -> ceil to 28 multiples -> 112x112 (no scaling, under 672). - let (h, w) = p.vllm_resize(100, 100); + // 100x100 -> rounded to 28 multiples -> 112x112. + let (h, w) = p.smart_resize(100, 100).unwrap(); assert_eq!(h, 112); assert_eq!(w, 112); assert_eq!(h % 28, 0); @@ -382,7 +342,7 @@ mod tests { #[test] fn test_resize_exact_max_size() { let p = MiniMaxM3Processor::new(); - let (h, w) = p.vllm_resize(672, 672); + let (h, w) = p.smart_resize(672, 672).unwrap(); assert_eq!((h, w), (672, 672)); // Sanity: 672x672 -> grid 48x48 -> 576 tokens == config image_seq_length. let (t, gh, gw) = p.calculate_grid_thw(h, w, 1); @@ -390,44 +350,27 @@ mod tests { } #[test] - fn test_resize_default_no_clamp() { - // Default (max_size=None) matches the live HF processor: dimensions are - // only rounded up to the factor, never scaled down. These values were - // verified against HF transformers golden output. + fn test_resize_scales_down_by_max_pixels() { let p = MiniMaxM3Processor::new(); // 500x500 -> 504x504 -> grid 36x36 -> 324 tokens. - assert_eq!(p.vllm_resize(500, 500), (504, 504)); - // 4000x3000 (w x h): ceil-aligned, NOT clamped -> grid 216x286. - let (h, w) = p.vllm_resize(3000, 4000); - assert_eq!((h, w), (3024, 4004)); + assert_eq!(p.smart_resize(500, 500).unwrap(), (504, 504)); + // 4000x3000 (w x h): scaled by max_pixels -> grid 40x54. + let (h, w) = p.smart_resize(3000, 4000).unwrap(); + assert_eq!((h, w), (560, 756)); let (t, gh, gw) = p.calculate_grid_thw(h, w, 1); - assert_eq!((gh, gw), (216, 286)); - assert_eq!(p.calculate_tokens_from_grid(t, gh, gw), 15444); - } - - #[test] - fn test_resize_with_clamp_scales_down() { - // With an explicit bound, large images are scaled down (the source's - // intended `size`-derived behavior). - let p = MiniMaxM3Processor::with_config(14, 2, 2, Some((672, 672))); - // 800x600 (w x h). ceil -> 812x616, exceeds max_w=672 -> scale down. - let (h, w) = p.vllm_resize(600, 800); - assert_eq!((h, w), (532, 672)); - assert!(w <= 672 && h <= 672); - assert_eq!(h % 28, 0); - assert_eq!(w % 28, 0); + assert_eq!((gh, gw), (40, 54)); + assert_eq!(p.calculate_tokens_from_grid(t, gh, gw), 540); } #[test] fn test_calculate_num_tokens() { let config = PreProcessorConfig::default(); - // Default (no clamp): 500x500 -> 324 tokens (verified against HF). + // 500x500 -> 324 tokens (verified against HF). let p = MiniMaxM3Processor::new(); assert_eq!(p.calculate_num_tokens(500, 500, &config), 324); - // With clamp: 800x600 -> grid 38x48 -> (38*48)/4 = 456. - let pc = MiniMaxM3Processor::with_config(14, 2, 2, Some((672, 672))); - assert_eq!(pc.calculate_num_tokens(800, 600, &config), 456); + // 4000x3000 -> grid 40x54 -> (40*54)/4 = 540. + assert_eq!(p.calculate_num_tokens(4000, 3000, &config), 540); } #[test] @@ -506,7 +449,6 @@ mod tests { let p = MiniMaxM3Processor::from_preprocessor_config(&config); assert_eq!(p.patch_size(), 14); assert_eq!(p.merge_size(), 2); - // No clamp by default — matches the live HF processor. - assert_eq!(p.max_size(), None); + assert_eq!(p.max_pixels(), DEFAULT_MAX_PIXELS); } } diff --git a/src/vision/processors/mod.rs b/src/vision/processors/mod.rs index 310bbcf..3d48f24 100644 --- a/src/vision/processors/mod.rs +++ b/src/vision/processors/mod.rs @@ -14,7 +14,7 @@ //! - **Phi4-Vision** (`phi4_vision`): Dynamic HD transform with 448x448 tiles and SiGLIP encoder //! - **LLaMA 4 Vision** (`llama4_vision`): Tile-based processing with 336x336 tiles and global tile //! - **Pixtral/Mistral3** (`pixtral`): CLIP-based preprocessing with dynamic resolution -//! - **MiniMax-M3** (`minimax_m3`): Qwen2-VL patchify with vLLM-style resize (max_size bound) +//! - **MiniMax-M3** (`minimax_m3`): Qwen2-VL patchify with MiniMax smart resize pub mod kimi_k25; pub mod llama4_vision; diff --git a/tests/vision_golden_tests.rs b/tests/vision_golden_tests.rs index 98290f6..0e2eb6b 100644 --- a/tests/vision_golden_tests.rs +++ b/tests/vision_golden_tests.rs @@ -8,7 +8,7 @@ //! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad) //! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models) //! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models) -//! - `minimax_m3/` - Qwen2-VL patchify with vLLM-style resize (MiniMaxAI/Minimax-M3-* models) +//! - `minimax_m3/` - Qwen2-VL patchify with MiniMax smart resize (MiniMaxAI/MiniMax-M3 models) //! //! To regenerate golden outputs: //! ```bash @@ -634,17 +634,15 @@ fn test_qwen3_vl_golden_grayscale() { /// 3. Pixel values match after patchification /// /// MiniMax-M3 shares Qwen2-VL's patchify pipeline (patch_size=14, merge_size=2, -/// CLIP normalization) but uses vLLM-style resize bounded by `max_size` instead -/// of Qwen's min/max-pixel smart resize. +/// CLIP normalization) with MiniMax's max-pixel smart resize setting. fn run_minimax_m3_golden_test(image_name: &str) { - let golden_dir = Path::new("crates/multimodal/tests/fixtures/golden/minimax_m3"); - let image_path = - Path::new("crates/multimodal/tests/fixtures/images").join(format!("{image_name}.jpg")); + let golden_dir = Path::new("tests/fixtures/golden/minimax_m3"); + let image_path = Path::new("tests/fixtures/images").join(format!("{image_name}.jpg")); if !golden_dir.exists() || !image_path.exists() { - eprintln!("Golden test fixtures for minimax_m3/{image_name} not found, skipping test"); - eprintln!( - "Run: python crates/multimodal/scripts/generate_vision_golden.py --model minimax_m3" + skip_missing_fixture( + format!("Golden test fixtures for minimax_m3/{image_name} not found"), + "python scripts/generate_vision_golden.py --model minimax_m3", ); return; }