diff --git a/.github/workflows/golden-fixtures.yml b/.github/workflows/golden-fixtures.yml index 646701f..7b2719f 100644 --- a/.github/workflows/golden-fixtures.yml +++ b/.github/workflows/golden-fixtures.yml @@ -46,6 +46,9 @@ jobs: - model: pixtral test_filter: test_pixtral_golden extra_filter: "" + - model: minimax_m3 + test_filter: test_minimax_m3_golden + extra_filter: "" steps: - uses: actions/checkout@v6 - uses: actions-rust-lang/setup-rust-toolchain@v1 diff --git a/Cargo.toml b/Cargo.toml index abd27a3..3f68736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ once_cell = "1.21.3" reqwest = { version = "0.12.8", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["preserve_order"] } +serde_with = "3.12" thiserror = "2.0.12" tokio = { version = "1.42.0", features = ["sync", "fs", "rt-multi-thread"] } url = "2.5.4" diff --git a/scripts/generate_vision_golden.py b/scripts/generate_vision_golden.py index 5b473f2..5ed7e8d 100755 --- a/scripts/generate_vision_golden.py +++ b/scripts/generate_vision_golden.py @@ -72,6 +72,11 @@ "processor_class": "PixtralImageProcessor", "description": "Dynamic resolution with CLIP normalization and bicubic resize", }, + "minimax_m3": { + "model_id": "MiniMaxAI/MiniMax-M3", + "processor_class": "MiniMaxM3VLImageProcessor", + "description": "Qwen2-VL patchify with MiniMax smart resize, CLIP normalization", + }, } # Default test images @@ -607,6 +612,79 @@ def generate_golden_pixtral(image_path: str, output_dir: str) -> dict: return result +def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict: + """Generate golden output for MiniMax-M3 VL. + + The patchify pipeline is identical to Qwen2-VL, with MiniMax's smart resize + defaults: + 1. Smart resize to fit within min/max pixel bounds + 2. Align dimensions to (patch_size * merge_size) boundary + 3. Normalize with CLIP mean/std + 4. Returns image_grid_thw for position encoding + + Default parameters: + - patch_size: 14 + - merge_size: 2 + - temporal_patch_size: 2 + - min_pixels: 4 * 28 * 28 = 3,136 + - max_pixels: 672 * 672 = 451,584 + """ + from transformers import AutoImageProcessor + + # The image processor is registered under AutoImageProcessor in the model's + # auto_map; instantiate it directly to avoid the tokenizer dependency of the + # full AutoProcessor. + img_processor = AutoImageProcessor.from_pretrained( + "MiniMaxAI/MiniMax-M3", trust_remote_code=True + ) + image = Image.open(image_path).convert("RGB") + original_size = image.size + + # Process image + outputs = img_processor(images=image, return_tensors="pt") + + # Convert to numpy for saving + pixel_values = outputs["pixel_values"].numpy() + image_grid_thw = outputs.get("image_grid_thw") + if image_grid_thw is not None: + image_grid_thw = image_grid_thw.numpy() + + # Get config values + patch_size = getattr(img_processor, "patch_size", 14) + merge_size = getattr(img_processor, "merge_size", 2) + temporal_patch_size = getattr(img_processor, "temporal_patch_size", 2) + + # Calculate number of tokens: (T * H * W) / merge_size² + if image_grid_thw is not None: + grid_thw = image_grid_thw[0] + num_tokens = int(np.prod(grid_thw) / (merge_size**2)) + else: + num_tokens = None + + result = { + "pixel_values": pixel_values, + "original_size": original_size, + "processor_config": img_processor.to_dict(), + } + + if image_grid_thw is not None: + result["image_grid_thw"] = image_grid_thw + + if num_tokens is not None: + result["num_tokens"] = num_tokens + + # Add debug info + result["config_info"] = { + "patch_size": patch_size, + "merge_size": merge_size, + "temporal_patch_size": temporal_patch_size, + "min_pixels": getattr(img_processor, "min_pixels", None), + "max_pixels": getattr(img_processor, "max_pixels", None), + } + + return result + + def generate_for_model(model_key: str, image_paths: list, output_dir: str): """Generate golden outputs for a specific model.""" print(f"\nGenerating golden outputs for {model_key}...") @@ -621,6 +699,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str): "phi4_vision": generate_golden_phi4_vision, "llama4_vision": generate_golden_llama4_vision, "pixtral": generate_golden_pixtral, + "minimax_m3": generate_golden_minimax_m3, }.get(model_key) if generator_fn is None: diff --git a/src/registry/minimax_m3.rs b/src/registry/minimax_m3.rs new file mode 100644 index 0000000..c3722e4 --- /dev/null +++ b/src/registry/minimax_m3.rs @@ -0,0 +1,168 @@ +use std::collections::HashMap; + +use serde_json::{json, Value}; + +use crate::{ + registry::{ModelMetadata, ModelProcessorSpec, RegistryResult}, + types::{FieldLayout, Modality, PromptReplacement, TokenId}, + vision::image_processor::PreprocessedImages, +}; + +pub(super) struct MiniMaxM3VisionSpec; + +impl MiniMaxM3VisionSpec { + const IMAGE_TOKEN: &'static str = "]<]image[>["; + const VISION_START_TOKEN: &'static str = "]<]start of image[>["; + const VISION_END_TOKEN: &'static str = "]<]end of image[>["; +} + +impl ModelProcessorSpec for MiniMaxM3VisionSpec { + fn name(&self) -> &'static str { + "minimax_m3_vl" + } + + fn matches(&self, metadata: &ModelMetadata) -> bool { + let id = metadata.model_id.to_ascii_lowercase(); + id.contains("minimax") && id.contains("m3") + || metadata + .config_model_type() + .is_some_and(|mt| mt == "minimax_m3_vl") + } + + fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult { + Ok(Self::IMAGE_TOKEN.to_string()) + } + + fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult { + metadata.token_id(Self::IMAGE_TOKEN) + } + + fn modality_limits( + &self, + _metadata: &ModelMetadata, + ) -> RegistryResult> { + Ok(HashMap::from([(Modality::Image, 64)])) + } + + fn processor_kwargs(&self, _metadata: &ModelMetadata) -> RegistryResult { + Ok(json!({})) + } + + fn prompt_replacements( + &self, + metadata: &ModelMetadata, + preprocessed: &PreprocessedImages, + ) -> RegistryResult> { + let image_token_id = metadata.token_id(Self::IMAGE_TOKEN)?; + let start_token_id = metadata.token_id(Self::VISION_START_TOKEN)?; + let end_token_id = metadata.token_id(Self::VISION_END_TOKEN)?; + + Ok(preprocessed + .num_img_tokens + .iter() + .map(|&n| { + // Mirrors MiniMaxM3VLMultiModalProcessor._get_prompt_updates: + // full = [start_token_id] + [image_token_id] * N + [end_token_id] + // The image_token_id positions are marked as embed tokens. + let mut tokens = Vec::with_capacity(n + 2); + tokens.push(start_token_id); + tokens.extend(std::iter::repeat_n(image_token_id, n)); + tokens.push(end_token_id); + PromptReplacement::sequence(Modality::Image, Self::IMAGE_TOKEN, tokens) + }) + .collect()) + } + + fn field_layouts(&self) -> HashMap { + HashMap::from([ + ( + "pixel_values".to_string(), + FieldLayout::flat("patches_per_image"), + ), + ("image_grid_thw".to_string(), FieldLayout::Batched), + ("patches_per_image".to_string(), FieldLayout::Batched), + ]) + } + + fn keep_on_cpu_keys(&self) -> Vec { + vec!["image_grid_thw".to_string()] + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::{ + registry::{test_helpers::*, ModelMetadata, ModelRegistry}, + types::ImageSize, + }; + + #[test] + fn minimax_m3_matches_model_type() { + let tokenizer = TestTokenizer::new(&[ + ("]<]image[>[", 200025), + ("]<]start of image[>[", 200029), + ("]<]end of image[>[", 200030), + ]); + let config = json!({ "model_type": "minimax_m3_vl" }); + let metadata = ModelMetadata { + model_id: "some-custom-model", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec"); + assert_eq!(spec.name(), "minimax_m3_vl"); + } + + #[test] + fn minimax_m3_matches_model_id() { + let tokenizer = TestTokenizer::new(&[ + ("]<]image[>[", 200025), + ("]<]start of image[>[", 200029), + ("]<]end of image[>[", 200030), + ]); + let config = json!({ "model_type": "minimax_m3_vl" }); + let metadata = ModelMetadata { + model_id: "MiniMaxAI/MiniMax-M3", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec"); + assert_eq!(spec.name(), "minimax_m3_vl"); + } + + #[test] + fn minimax_m3_prompt_replacements_wrap_with_start_end() { + let tokenizer = TestTokenizer::new(&[ + ("]<]image[>[", 200025), + ("]<]start of image[>[", 200029), + ("]<]end of image[>[", 200030), + ]); + let config = json!({ "model_type": "minimax_m3_vl" }); + let metadata = ModelMetadata { + model_id: "MiniMaxAI/MiniMax-M3", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec"); + + // 672x672 image → 576 tokens (48x48 grid, merge_size=2 → 48*48/4=576) + let replacements = spec + .prompt_replacements( + &metadata, + &test_preprocessed_with_tokens(&[ImageSize::new(672, 672)], &[576]), + ) + .unwrap(); + + assert_eq!(replacements.len(), 1); + let tokens = &replacements[0].tokens; + assert_eq!(tokens.len(), 578); // 1 start + 576 image + 1 end + assert_eq!(tokens[0], 200029); // start + assert!(tokens[1..577].iter().all(|&t| t == 200025)); // image tokens + assert_eq!(tokens[577], 200030); // end + } +} diff --git a/src/registry/mod.rs b/src/registry/mod.rs index e0cbc28..0554b65 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -1,6 +1,7 @@ mod kimi_k25; mod llama4; mod llava; +mod minimax_m3; mod phi3_v; mod qwen3_vl; mod qwen_vl; @@ -9,6 +10,7 @@ mod traits; use kimi_k25::KimiK25VisionSpec; use llama4::Llama4Spec; use llava::{LlavaNextSpec, LlavaSpec}; +use minimax_m3::MiniMaxM3VisionSpec; use once_cell::sync::Lazy; use phi3_v::Phi3VisionSpec; use qwen3_vl::Qwen3VLVisionSpec; @@ -31,6 +33,7 @@ impl ModelRegistry { // LlavaNext must be registered before Llava so "llava_next" model_type matches first. LazySpec::new("llava_next", || Box::new(LlavaNextSpec)), LazySpec::new("llava", || Box::new(LlavaSpec)), + LazySpec::new("minimax_m3_vl", || Box::new(MiniMaxM3VisionSpec)), // Qwen3-VL must be registered before QwenVL so "qwen3" matches first. LazySpec::new("qwen3_vl", || Box::new(Qwen3VLVisionSpec)), LazySpec::new("qwen_vl", || Box::new(QwenVLVisionSpec)), diff --git a/src/vision/image_processor.rs b/src/vision/image_processor.rs index 97cc870..c4a0c38 100644 --- a/src/vision/image_processor.rs +++ b/src/vision/image_processor.rs @@ -474,6 +474,16 @@ impl ImageProcessorRegistry { Box::new(super::processors::KimiK25Processor::new()), ); + // Register MiniMax-M3 VL + registry.register( + "minimax-m3-vl", + Box::new(super::processors::MiniMaxM3Processor::new()), + ); + registry.register( + "minimax_m3_vl", + Box::new(super::processors::MiniMaxM3Processor::new()), + ); + registry } } diff --git a/src/vision/mod.rs b/src/vision/mod.rs index f980b1a..273195d 100644 --- a/src/vision/mod.rs +++ b/src/vision/mod.rs @@ -40,7 +40,7 @@ pub use image_processor::{ }; pub use preprocessor_config::PreProcessorConfig; pub use processors::{ - Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, Phi3VisionProcessor, - Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor, + Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, MiniMaxM3Processor, + Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor, }; pub use transforms::TransformError; diff --git a/src/vision/preprocessor_config.rs b/src/vision/preprocessor_config.rs index d03251d..560fd71 100644 --- a/src/vision/preprocessor_config.rs +++ b/src/vision/preprocessor_config.rs @@ -7,9 +7,37 @@ use std::collections::HashMap; use image::imageops::FilterType; use serde::{Deserialize, Deserializer}; +use serde_with::{serde_as, TryFromInto}; use super::transforms; +/// Deserialization shape for HF size fields. +/// +/// - Map format: `"size": {"height": 672, "width": 672}` (standard HuggingFace) +/// - Array format: `"size": [672, 672]` -> treated as [height, width] (MiniMax M3) +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum SizeRepr { + Map(HashMap), + Array([u32; 2]), +} + +impl TryFrom for HashMap { + type Error = &'static str; + + fn try_from(value: SizeRepr) -> Result { + match value { + SizeRepr::Map(size) => Ok(size), + SizeRepr::Array([height, width]) => { + let mut size = HashMap::new(); + size.insert("height".to_string(), height); + size.insert("width".to_string(), width); + Ok(size) + } + } + } +} + /// Struct to represent patch_size as dict {"height": x, "width": y} #[derive(Debug, Clone, Deserialize, Default)] pub struct PatchSize { @@ -101,6 +129,7 @@ where /// /// This struct captures the common fields across different vision model processors. /// Model-specific fields are accessed via the flexible `extra` field. +#[serde_as] #[derive(Debug, Clone, Deserialize, Default)] pub struct PreProcessorConfig { /// Processor class name (e.g., "CLIPImageProcessor", "Qwen2VLImageProcessor") @@ -148,11 +177,13 @@ pub struct PreProcessorConfig { pub resampling: Option, /// Target size for resizing - /// Can be {"height": H, "width": W} or {"shortest_edge": S} + /// Can be {"height": H, "width": W}, {"shortest_edge": S}, or [H, W] + #[serde_as(deserialize_as = "Option>")] #[serde(default)] pub size: Option>, /// Target size for center cropping + #[serde_as(deserialize_as = "Option>")] #[serde(default)] pub crop_size: Option>, @@ -475,6 +506,11 @@ mod tests { let json2 = r#"{"size": {"shortest_edge": 224}}"#; let config2 = PreProcessorConfig::from_json(json2).unwrap(); assert_eq!(config2.get_target_size(), Some((224, 224))); + + // Array format [height, width] (e.g. MiniMax M3) + let json3 = r#"{"size": [672, 672]}"#; + let config3 = PreProcessorConfig::from_json(json3).unwrap(); + assert_eq!(config3.get_target_size(), Some((672, 672))); } #[test] diff --git a/src/vision/processors/minimax_m3.rs b/src/vision/processors/minimax_m3.rs new file mode 100644 index 0000000..9b0c3c0 --- /dev/null +++ b/src/vision/processors/minimax_m3.rs @@ -0,0 +1,454 @@ +//! MiniMax-M3 VL image processor. +//! +//! Ported from HuggingFace `MiniMaxM3VLImageProcessor`. The patchify pipeline +//! (rescale → normalize → reshape into `[grid_t, grid_h, grid_w, ...]` patches) +//! is identical to Qwen2-VL, so we reuse [`QwenVLProcessorBase`] for it. +//! +//! MiniMax-M3 uses Qwen-style smart resize with `max_pixels = 672 * 672`. + +use image::{imageops::FilterType, DynamicImage, GenericImageView}; + +use super::qwen_vl_base::{QwenVLConfig, QwenVLProcessorBase}; +use crate::vision::{ + image_processor::{ImagePreProcessor, ModelSpecificValue, PreprocessedImages}, + preprocessor_config::PreProcessorConfig, + transforms::{pil_to_filter, resize, to_tensor, to_tensor_and_normalize, TransformError}, +}; + +/// CLIP normalization mean values used by MiniMax-M3 VL. +pub const MINIMAX_M3_MEAN: [f64; 3] = [0.48145466, 0.4578275, 0.40821073]; + +/// CLIP normalization std values used by MiniMax-M3 VL. +pub const MINIMAX_M3_STD: [f64; 3] = [0.26862954, 0.26130258, 0.27577711]; + +/// Default vision encoder patch size. +pub const DEFAULT_PATCH_SIZE: usize = 14; + +/// Default spatial merge size (token reduction). +pub const DEFAULT_MERGE_SIZE: usize = 2; + +/// Default temporal patch size (for video frames; images repeat the single frame). +pub const DEFAULT_TEMPORAL_PATCH_SIZE: usize = 2; + +/// Default minimum pixels (4 * 28 * 28 = 3,136). +pub const DEFAULT_MIN_PIXELS: usize = 4 * 28 * 28; + +/// Default maximum pixels (672 * 672 = 451,584). +pub const DEFAULT_MAX_PIXELS: usize = 672 * 672; + +/// MiniMax-M3 VL image processor. +/// +/// Wraps [`QwenVLProcessorBase`] for the shared smart-resize, patchify, and grid +/// logic. +#[derive(Debug, Clone)] +pub struct MiniMaxM3Processor { + inner: QwenVLProcessorBase, +} + +impl Default for MiniMaxM3Processor { + fn default() -> Self { + Self::new() + } +} + +impl MiniMaxM3Processor { + /// Create a new MiniMax-M3 processor with default settings. + /// + /// Defaults: + /// - patch_size: 14 + /// - merge_size: 2 + /// - temporal_patch_size: 2 + /// - min_pixels: 3,136 + /// - max_pixels: 451,584 + /// - normalization: CLIP mean/std + pub fn new() -> Self { + Self::with_config( + DEFAULT_PATCH_SIZE, + DEFAULT_MERGE_SIZE, + DEFAULT_TEMPORAL_PATCH_SIZE, + DEFAULT_MIN_PIXELS, + DEFAULT_MAX_PIXELS, + ) + } + + /// Create a processor with custom settings. + pub fn with_config( + patch_size: usize, + merge_size: usize, + temporal_patch_size: usize, + min_pixels: usize, + max_pixels: usize, + ) -> Self { + Self { + inner: QwenVLProcessorBase::new(QwenVLConfig { + patch_size, + merge_size, + min_pixels, + max_pixels, + temporal_patch_size, + mean: MINIMAX_M3_MEAN, + std: MINIMAX_M3_STD, + model_name: "minimax-m3", + }), + } + } + + /// Create a processor from a HuggingFace preprocessor config. + pub fn from_preprocessor_config(config: &PreProcessorConfig) -> Self { + let patch_size = config.get_patch_size(DEFAULT_PATCH_SIZE); + let merge_size = config.merge_size.unwrap_or(DEFAULT_MERGE_SIZE); + let temporal_patch_size = config + .temporal_patch_size + .unwrap_or(DEFAULT_TEMPORAL_PATCH_SIZE); + let min_pixels = config.min_pixels.unwrap_or(DEFAULT_MIN_PIXELS); + let max_pixels = config.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS); + Self::with_config( + patch_size, + merge_size, + temporal_patch_size, + min_pixels, + max_pixels, + ) + } + + /// Get the patch size. + pub fn patch_size(&self) -> usize { + self.inner.patch_size() + } + + /// Get the merge size. + pub fn merge_size(&self) -> usize { + self.inner.merge_size() + } + + /// Get the temporal patch size. + pub fn temporal_patch_size(&self) -> usize { + self.inner.temporal_patch_size() + } + + /// Get the minimum pixels. + pub fn min_pixels(&self) -> usize { + self.inner.min_pixels() + } + + /// Get the maximum pixels. + pub fn max_pixels(&self) -> usize { + self.inner.max_pixels() + } + + /// Get the factor for dimension alignment (`patch_size * merge_size`). + #[inline] + pub fn get_factor(&self) -> usize { + self.inner.get_factor() + } + + /// Smart resize. Returns `(new_height, new_width)`. + pub fn smart_resize( + &self, + height: usize, + width: usize, + ) -> Result<(usize, usize), TransformError> { + self.inner.smart_resize(height, width) + } + + /// Calculate the grid dimensions `(grid_t, grid_h, grid_w)` for an image. + pub fn calculate_grid_thw( + &self, + height: usize, + width: usize, + num_frames: usize, + ) -> (usize, usize, usize) { + self.inner.calculate_grid_thw(height, width, num_frames) + } + + /// Calculate the number of image tokens after merge. + pub fn calculate_tokens_from_grid(&self, grid_t: usize, grid_h: usize, grid_w: usize) -> usize { + self.inner + .calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + /// Pick the resampling filter, defaulting to BICUBIC (the HF default) when the + /// config doesn't specify one. + fn resize_filter(config: &PreProcessorConfig) -> FilterType { + match config.resampling { + Some(r) => pil_to_filter(Some(r)), + None => FilterType::CatmullRom, // BICUBIC + } + } +} + +impl ImagePreProcessor for MiniMaxM3Processor { + fn default_mean(&self) -> [f64; 3] { + MINIMAX_M3_MEAN + } + + fn default_std(&self) -> [f64; 3] { + MINIMAX_M3_STD + } + + fn preprocess( + &self, + images: &[DynamicImage], + config: &PreProcessorConfig, + ) -> Result { + if images.is_empty() { + return Err(TransformError::EmptyBatch); + } + + let image_sizes: Vec<(u32, u32)> = images.iter().map(|img| img.dimensions()).collect(); + + let mean = config.get_image_mean(); + let std = config.get_image_std(); + let filter = Self::resize_filter(config); + + let patch_size = self.patch_size(); + let temporal_patch_size = self.temporal_patch_size(); + let patch_features = 3 * temporal_patch_size * patch_size * patch_size; + + let mut all_patches: Vec = Vec::new(); + let mut patches_per_image: Vec = Vec::with_capacity(images.len()); + let mut grid_thw_data = Vec::with_capacity(images.len() * 3); + let mut num_img_tokens = Vec::with_capacity(images.len()); + + for image in images { + let (w, h) = image.dimensions(); + let (target_h, target_w) = self.smart_resize(h as usize, w as usize)?; + + // Resize to the image's own target size (skip if dimensions match). + let (tw32, th32) = (target_w as u32, target_h as u32); + let needs_resize = config.do_resize.unwrap_or(true) && (w != tw32 || h != th32); + let resized; + let img_ref = if needs_resize { + resized = resize(image, tw32, th32, filter); + &resized + } else { + image + }; + + // Grid dimensions based on the target size (T=1 for a single image). + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(target_h, target_w, 1); + grid_thw_data.push(grid_t as i64); + grid_thw_data.push(grid_h as i64); + grid_thw_data.push(grid_w as i64); + + let num_patches = grid_t * grid_h * grid_w; + num_img_tokens.push(self.calculate_tokens_from_grid(grid_t, grid_h, grid_w)); + + // Convert to tensor [C, H, W] (+ normalize) in one fused pass. + let tensor = if config.do_normalize.unwrap_or(true) { + to_tensor_and_normalize(img_ref, &mean, &std) + } else { + to_tensor(img_ref) + }; + + // Patchify directly into the shared buffer. The single image frame is + // repeated across `temporal_patch_size` inside `patchify_into`, which + // matches HF's "repeat last frame to fill the temporal dim". + self.inner + .patchify_into(&tensor, grid_t, grid_h, grid_w, &mut all_patches)?; + patches_per_image.push(num_patches as i64); + } + + let total_patches: usize = patches_per_image.iter().map(|&n| n as usize).sum(); + let pixel_values = + ndarray::Array2::from_shape_vec((total_patches, patch_features), all_patches).map_err( + |e| { + TransformError::ShapeError(format!( + "Failed to create patchified pixel_values [{total_patches}, {patch_features}]: {e}" + )) + }, + )?; + + let result = + PreprocessedImages::new_dynamic(pixel_values.into_dyn(), num_img_tokens, image_sizes) + .with_extra( + "image_grid_thw", + ModelSpecificValue::int_2d(grid_thw_data, images.len(), 3), + ) + .with_extra( + "patches_per_image", + ModelSpecificValue::int_1d(patches_per_image), + ); + + Ok(result) + } + + fn calculate_num_tokens(&self, width: u32, height: u32, _config: &PreProcessorConfig) -> usize { + let (new_height, new_width) = self + .smart_resize(height as usize, width as usize) + .unwrap_or_else(|_| { + let factor = self.get_factor(); + (factor, factor) + }); + let (grid_t, grid_h, grid_w) = self.calculate_grid_thw(new_height, new_width, 1); + self.calculate_tokens_from_grid(grid_t, grid_h, grid_w) + } + + fn model_name(&self) -> &'static str { + "minimax-m3" + } + + fn get_processed_size(&self, _config: &PreProcessorConfig) -> Option<(u32, u32)> { + // Dynamic resolution: no fixed output size. + None + } +} + +#[cfg(test)] +mod tests { + use image::{Rgb, RgbImage}; + + use super::*; + use crate::vision::image_processor::ModelSpecificValue; + + fn create_test_image(width: u32, height: u32, color: Rgb) -> DynamicImage { + DynamicImage::from(RgbImage::from_pixel(width, height, color)) + } + + #[test] + fn test_defaults() { + let p = MiniMaxM3Processor::new(); + assert_eq!(p.patch_size(), 14); + assert_eq!(p.merge_size(), 2); + assert_eq!(p.temporal_patch_size(), 2); + assert_eq!(p.get_factor(), 28); // 14 * 2 + assert_eq!(p.min_pixels(), DEFAULT_MIN_PIXELS); + assert_eq!(p.max_pixels(), DEFAULT_MAX_PIXELS); + } + + #[test] + fn test_mean_std() { + let p = MiniMaxM3Processor::new(); + assert_eq!(p.default_mean(), MINIMAX_M3_MEAN); + assert_eq!(p.default_std(), MINIMAX_M3_STD); + } + + #[test] + fn test_model_name() { + assert_eq!(MiniMaxM3Processor::new().model_name(), "minimax-m3"); + } + + #[test] + fn test_resize_within_bounds_aligns_up() { + let p = MiniMaxM3Processor::new(); + // 100x100 -> rounded to 28 multiples -> 112x112. + let (h, w) = p.smart_resize(100, 100).unwrap(); + assert_eq!(h, 112); + assert_eq!(w, 112); + assert_eq!(h % 28, 0); + assert_eq!(w % 28, 0); + } + + #[test] + fn test_resize_exact_max_size() { + let p = MiniMaxM3Processor::new(); + let (h, w) = p.smart_resize(672, 672).unwrap(); + assert_eq!((h, w), (672, 672)); + // Sanity: 672x672 -> grid 48x48 -> 576 tokens == config image_seq_length. + let (t, gh, gw) = p.calculate_grid_thw(h, w, 1); + assert_eq!(p.calculate_tokens_from_grid(t, gh, gw), 576); + } + + #[test] + fn test_resize_scales_down_by_max_pixels() { + let p = MiniMaxM3Processor::new(); + + // 500x500 -> 504x504 -> grid 36x36 -> 324 tokens. + assert_eq!(p.smart_resize(500, 500).unwrap(), (504, 504)); + // 4000x3000 (w x h): scaled by max_pixels -> grid 40x54. + let (h, w) = p.smart_resize(3000, 4000).unwrap(); + assert_eq!((h, w), (560, 756)); + let (t, gh, gw) = p.calculate_grid_thw(h, w, 1); + assert_eq!((gh, gw), (40, 54)); + assert_eq!(p.calculate_tokens_from_grid(t, gh, gw), 540); + } + + #[test] + fn test_calculate_num_tokens() { + let config = PreProcessorConfig::default(); + // 500x500 -> 324 tokens (verified against HF). + let p = MiniMaxM3Processor::new(); + assert_eq!(p.calculate_num_tokens(500, 500, &config), 324); + // 4000x3000 -> grid 40x54 -> (40*54)/4 = 540. + assert_eq!(p.calculate_num_tokens(4000, 3000, &config), 540); + } + + #[test] + fn test_preprocess_single() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig { + do_resize: Some(true), + do_normalize: Some(true), + image_mean: Some(MINIMAX_M3_MEAN.to_vec()), + image_std: Some(MINIMAX_M3_STD.to_vec()), + ..Default::default() + }; + + let image = create_test_image(600, 400, Rgb([128, 128, 128])); + let result = p.preprocess(&[image], &config).unwrap(); + + // pixel_values is patchified: [total_patches, patch_features]. + assert_eq!(result.pixel_values.ndim(), 2); + assert_eq!(result.pixel_values.shape()[1], 3 * 2 * 14 * 14); // 1176 + assert!(result.pixel_values.shape()[0] > 0); + + assert!(result.model_specific.contains_key("image_grid_thw")); + assert!(result.model_specific.contains_key("patches_per_image")); + assert!(result.num_img_tokens[0] > 0); + } + + #[test] + fn test_preprocess_multiple() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig::default(); + + let images = vec![ + create_test_image(600, 400, Rgb([100, 100, 100])), + create_test_image(400, 600, Rgb([150, 150, 150])), + ]; + + let result = p.preprocess(&images, &config).unwrap(); + + assert_eq!(result.image_sizes.len(), 2); + assert_eq!(result.num_img_tokens.len(), 2); + assert_eq!(result.pixel_values.ndim(), 2); + + if let Some(ModelSpecificValue::IntTensor { data, shape }) = + result.model_specific.get("image_grid_thw") + { + assert_eq!(shape, &[2, 3]); + assert_eq!(data.len(), 6); + } else { + panic!("Expected image_grid_thw to be IntTensor"); + } + + if let Some(ModelSpecificValue::IntTensor { data, .. }) = + result.model_specific.get("patches_per_image") + { + let total: i64 = data.iter().sum(); + assert_eq!(total as usize, result.pixel_values.shape()[0]); + } else { + panic!("Expected patches_per_image to be IntTensor"); + } + } + + #[test] + fn test_preprocess_empty_batch_errors() { + let p = MiniMaxM3Processor::new(); + let config = PreProcessorConfig::default(); + assert!(p.preprocess(&[], &config).is_err()); + } + + #[test] + fn test_from_preprocessor_config() { + let config = PreProcessorConfig { + merge_size: Some(2), + temporal_patch_size: Some(2), + ..Default::default() + }; + let p = MiniMaxM3Processor::from_preprocessor_config(&config); + assert_eq!(p.patch_size(), 14); + assert_eq!(p.merge_size(), 2); + assert_eq!(p.max_pixels(), DEFAULT_MAX_PIXELS); + } +} diff --git a/src/vision/processors/mod.rs b/src/vision/processors/mod.rs index eeded26..3d48f24 100644 --- a/src/vision/processors/mod.rs +++ b/src/vision/processors/mod.rs @@ -14,10 +14,12 @@ //! - **Phi4-Vision** (`phi4_vision`): Dynamic HD transform with 448x448 tiles and SiGLIP encoder //! - **LLaMA 4 Vision** (`llama4_vision`): Tile-based processing with 336x336 tiles and global tile //! - **Pixtral/Mistral3** (`pixtral`): CLIP-based preprocessing with dynamic resolution +//! - **MiniMax-M3** (`minimax_m3`): Qwen2-VL patchify with MiniMax smart resize pub mod kimi_k25; pub mod llama4_vision; pub mod llava; +pub mod minimax_m3; pub mod phi3_vision; pub mod phi4_vision; pub mod pixtral; @@ -28,6 +30,7 @@ pub mod qwen_vl_base; pub use kimi_k25::KimiK25Processor; pub use llama4_vision::Llama4VisionProcessor; pub use llava::{ImageAspectRatio, LlavaNextProcessor, LlavaProcessor}; +pub use minimax_m3::MiniMaxM3Processor; pub use phi3_vision::Phi3VisionProcessor; pub use phi4_vision::Phi4VisionProcessor; pub use pixtral::PixtralProcessor; diff --git a/tests/vision_golden_tests.rs b/tests/vision_golden_tests.rs index b493b84..0e2eb6b 100644 --- a/tests/vision_golden_tests.rs +++ b/tests/vision_golden_tests.rs @@ -8,6 +8,7 @@ //! - `llava_pad/` - Expand-to-square mode (liuhaotian/llava-* models, image_aspect_ratio=pad) //! - `qwen2_vl/` - Dynamic resolution with smart resize (Qwen/Qwen2-VL-* models) //! - `qwen3_vl/` - Dynamic resolution with patch_size=16 and [0.5,0.5,0.5] norm (Qwen/Qwen3-VL-* models) +//! - `minimax_m3/` - Qwen2-VL patchify with MiniMax smart resize (MiniMaxAI/MiniMax-M3 models) //! //! To regenerate golden outputs: //! ```bash @@ -32,8 +33,8 @@ use std::{fs::File, io::Read, path::Path}; use llm_multimodal::vision::{ image_processor::ModelSpecificValue, ImagePreProcessor, Llama4VisionProcessor, LlavaProcessor, - Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, PreProcessorConfig, - Qwen2VLProcessor, Qwen3VLProcessor, + MiniMaxM3Processor, Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, + PreProcessorConfig, Qwen2VLProcessor, Qwen3VLProcessor, }; use ndarray::{Array4, Array5}; @@ -621,6 +622,152 @@ fn test_qwen3_vl_golden_grayscale() { run_qwen3_vl_golden_test("grayscale"); } +// ============================================================================ +// MiniMax-M3 tests +// ============================================================================ + +/// Run a MiniMax-M3 golden test for a specific image. +/// +/// This test validates: +/// 1. image_grid_thw matches the HuggingFace output +/// 2. num_tokens calculation is correct +/// 3. Pixel values match after patchification +/// +/// MiniMax-M3 shares Qwen2-VL's patchify pipeline (patch_size=14, merge_size=2, +/// CLIP normalization) with MiniMax's max-pixel smart resize setting. +fn run_minimax_m3_golden_test(image_name: &str) { + let golden_dir = Path::new("tests/fixtures/golden/minimax_m3"); + let image_path = Path::new("tests/fixtures/images").join(format!("{image_name}.jpg")); + + if !golden_dir.exists() || !image_path.exists() { + skip_missing_fixture( + format!("Golden test fixtures for minimax_m3/{image_name} not found"), + "python scripts/generate_vision_golden.py --model minimax_m3", + ); + return; + } + + let npz_path = golden_dir.join(format!("golden_{image_name}.npz")); + let config = load_config(&golden_dir.join("preprocessor_config.json")); + + // Load golden values + let golden_grid_thw = load_golden_grid_thw(&npz_path); + let golden_num_tokens = load_golden_num_tokens(&npz_path); + let (golden_pixels, golden_shape) = load_golden_qwen2_vl_pixels(&npz_path); + + // Process image with our Rust processor + let image = image::open(&image_path).expect("Failed to open image"); + let processor = MiniMaxM3Processor::from_preprocessor_config(&config); + let result = processor + .preprocess(&[image], &config) + .expect("Processing failed"); + + // Extract image_grid_thw from result + let rust_grid_thw = match result.model_specific.get("image_grid_thw") { + Some(ModelSpecificValue::IntTensor { data, shape }) => { + assert_eq!(shape, &[1, 3], "Expected shape [1, 3] for single image"); + data.clone() + } + _ => panic!("Expected image_grid_thw in model_specific"), + }; + + // Compare grid dimensions + println!( + "minimax_m3 - {image_name} image - Grid T H W: golden={golden_grid_thw:?}, rust={rust_grid_thw:?}" + ); + assert_eq!( + golden_grid_thw, rust_grid_thw, + "image_grid_thw mismatch for {image_name}" + ); + + // Compare token counts + let rust_num_tokens = result.num_img_tokens[0]; + println!( + "minimax_m3 - {image_name} image - Tokens: golden={golden_num_tokens}, rust={rust_num_tokens}" + ); + assert_eq!( + golden_num_tokens, rust_num_tokens, + "num_tokens mismatch for {image_name}" + ); + + // pixel_values is already patchified: [total_patches, patch_features] + let rust_patches = result.pixel_values_flat(); + let rust_shape = ( + result.pixel_values.shape()[0], + result.pixel_values.shape()[1], + ); + + println!( + "minimax_m3 - {image_name} image - Patch shape: golden={golden_shape:?}, rust={rust_shape:?}" + ); + assert_eq!(golden_shape, rust_shape, "Patch shape mismatch"); + + // Compare pixel values + let max_diff = rust_patches + .iter() + .zip(golden_pixels.iter()) + .map(|(r, g)| (r - g).abs()) + .fold(0.0f32, f32::max); + + println!("minimax_m3 - {image_name} image - Max pixel diff: {max_diff:.6}"); + + // Allow tolerance for floating point and interpolation differences + assert!( + max_diff < 0.1, + "Max pixel difference {max_diff} exceeds tolerance 0.1 for {image_name}" + ); +} + +#[test] +fn test_minimax_m3_golden_square() { + run_minimax_m3_golden_test("square"); +} + +#[test] +fn test_minimax_m3_golden_tall() { + run_minimax_m3_golden_test("tall"); +} + +#[test] +fn test_minimax_m3_golden_wide() { + run_minimax_m3_golden_test("wide"); +} + +#[test] +fn test_minimax_m3_golden_small() { + run_minimax_m3_golden_test("small"); +} + +#[test] +fn test_minimax_m3_golden_tiny() { + run_minimax_m3_golden_test("tiny"); +} + +#[test] +fn test_minimax_m3_golden_very_tall() { + run_minimax_m3_golden_test("very_tall"); +} + +#[test] +fn test_minimax_m3_golden_very_wide() { + run_minimax_m3_golden_test("very_wide"); +} + +#[test] +fn test_minimax_m3_golden_large() { + run_minimax_m3_golden_test("large"); +} + +#[test] +fn test_minimax_m3_golden_odd_dims() { + run_minimax_m3_golden_test("odd_dims"); +} + +#[test] +fn test_minimax_m3_golden_grayscale() { + run_minimax_m3_golden_test("grayscale"); +} + // ============================================================================ // Phi3-Vision tests // ============================================================================