feat: add Minimax-M3 support

.github/workflows/golden-fixtures.yml

@@ -46,6 +46,9 @@ jobs:
           - model: pixtral
             test_filter: test_pixtral_golden
             extra_filter: ""
+          - model: minimax_m3
+            test_filter: test_minimax_m3_golden
+            extra_filter: ""
     steps:
       - uses: actions/checkout@v6
       - uses: actions-rust-lang/setup-rust-toolchain@v1

Cargo.toml

@@ -30,6 +30,7 @@ once_cell = "1.21.3"
 reqwest = { version = "0.12.8", default-features = false }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["preserve_order"] }
+serde_with = "3.12"
 thiserror = "2.0.12"
 tokio = { version = "1.42.0", features = ["sync", "fs", "rt-multi-thread"] }
 url = "2.5.4"

scripts/generate_vision_golden.py

@@ -72,6 +72,11 @@
         "processor_class": "PixtralImageProcessor",
         "description": "Dynamic resolution with CLIP normalization and bicubic resize",
     },
+    "minimax_m3": {
+        "model_id": "MiniMaxAI/MiniMax-M3",
+        "processor_class": "MiniMaxM3VLImageProcessor",
+        "description": "Qwen2-VL patchify with MiniMax smart resize, CLIP normalization",
+    },
 }
 
 # Default test images
@@ -607,6 +612,79 @@ def generate_golden_pixtral(image_path: str, output_dir: str) -> dict:
     return result
 
 
+def generate_golden_minimax_m3(image_path: str, output_dir: str) -> dict:
+    """Generate golden output for MiniMax-M3 VL.
+
+    The patchify pipeline is identical to Qwen2-VL, with MiniMax's smart resize
+    defaults:
+    1. Smart resize to fit within min/max pixel bounds
+    2. Align dimensions to (patch_size * merge_size) boundary
+    3. Normalize with CLIP mean/std
+    4. Returns image_grid_thw for position encoding
+
+    Default parameters:
+    - patch_size: 14
+    - merge_size: 2
+    - temporal_patch_size: 2
+    - min_pixels: 4 * 28 * 28 = 3,136
+    - max_pixels: 672 * 672 = 451,584
+    """
+    from transformers import AutoImageProcessor
+
+    # The image processor is registered under AutoImageProcessor in the model's
+    # auto_map; instantiate it directly to avoid the tokenizer dependency of the
+    # full AutoProcessor.
+    img_processor = AutoImageProcessor.from_pretrained(
+        "MiniMaxAI/MiniMax-M3", trust_remote_code=True
+    )
+    image = Image.open(image_path).convert("RGB")
+    original_size = image.size
+
+    # Process image
+    outputs = img_processor(images=image, return_tensors="pt")
+
+    # Convert to numpy for saving
+    pixel_values = outputs["pixel_values"].numpy()
+    image_grid_thw = outputs.get("image_grid_thw")
+    if image_grid_thw is not None:
+        image_grid_thw = image_grid_thw.numpy()
+
+    # Get config values
+    patch_size = getattr(img_processor, "patch_size", 14)
+    merge_size = getattr(img_processor, "merge_size", 2)
+    temporal_patch_size = getattr(img_processor, "temporal_patch_size", 2)
+
+    # Calculate number of tokens: (T * H * W) / merge_size²
+    if image_grid_thw is not None:
+        grid_thw = image_grid_thw[0]
+        num_tokens = int(np.prod(grid_thw) / (merge_size**2))
+    else:
+        num_tokens = None
+
+    result = {
+        "pixel_values": pixel_values,
+        "original_size": original_size,
+        "processor_config": img_processor.to_dict(),
+    }
+
+    if image_grid_thw is not None:
+        result["image_grid_thw"] = image_grid_thw
+
+    if num_tokens is not None:
+        result["num_tokens"] = num_tokens
+
+    # Add debug info
+    result["config_info"] = {
+        "patch_size": patch_size,
+        "merge_size": merge_size,
+        "temporal_patch_size": temporal_patch_size,
+        "min_pixels": getattr(img_processor, "min_pixels", None),
+        "max_pixels": getattr(img_processor, "max_pixels", None),
+    }
+
+    return result
+
+
 def generate_for_model(model_key: str, image_paths: list, output_dir: str):
     """Generate golden outputs for a specific model."""
     print(f"\nGenerating golden outputs for {model_key}...")
@@ -621,6 +699,7 @@ def generate_for_model(model_key: str, image_paths: list, output_dir: str):
         "phi4_vision": generate_golden_phi4_vision,
         "llama4_vision": generate_golden_llama4_vision,
         "pixtral": generate_golden_pixtral,
+        "minimax_m3": generate_golden_minimax_m3,
     }.get(model_key)
 
     if generator_fn is None:

src/registry/minimax_m3.rs

@@ -0,0 +1,168 @@
+use std::collections::HashMap;
+
+use serde_json::{json, Value};
+
+use crate::{
+    registry::{ModelMetadata, ModelProcessorSpec, RegistryResult},
+    types::{FieldLayout, Modality, PromptReplacement, TokenId},
+    vision::image_processor::PreprocessedImages,
+};
+
+pub(super) struct MiniMaxM3VisionSpec;
+
+impl MiniMaxM3VisionSpec {
+    const IMAGE_TOKEN: &'static str = "]<]image[>[";
+    const VISION_START_TOKEN: &'static str = "]<]start of image[>[";
+    const VISION_END_TOKEN: &'static str = "]<]end of image[>[";
+}
+
+impl ModelProcessorSpec for MiniMaxM3VisionSpec {
+    fn name(&self) -> &'static str {
+        "minimax_m3_vl"
+    }
+
+    fn matches(&self, metadata: &ModelMetadata) -> bool {
+        let id = metadata.model_id.to_ascii_lowercase();
+        id.contains("minimax") && id.contains("m3")
+            || metadata
+                .config_model_type()
+                .is_some_and(|mt| mt == "minimax_m3_vl")
+    }
+
+    fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult<String> {
+        Ok(Self::IMAGE_TOKEN.to_string())
+    }
+
+    fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult<TokenId> {
+        metadata.token_id(Self::IMAGE_TOKEN)
+    }
+
+    fn modality_limits(
+        &self,
+        _metadata: &ModelMetadata,
+    ) -> RegistryResult<HashMap<Modality, usize>> {
+        Ok(HashMap::from([(Modality::Image, 64)]))
+    }
+
+    fn processor_kwargs(&self, _metadata: &ModelMetadata) -> RegistryResult<Value> {
+        Ok(json!({}))
+    }
+
+    fn prompt_replacements(
+        &self,
+        metadata: &ModelMetadata,
+        preprocessed: &PreprocessedImages,
+    ) -> RegistryResult<Vec<PromptReplacement>> {
+        let image_token_id = metadata.token_id(Self::IMAGE_TOKEN)?;
+        let start_token_id = metadata.token_id(Self::VISION_START_TOKEN)?;
+        let end_token_id = metadata.token_id(Self::VISION_END_TOKEN)?;
+
+        Ok(preprocessed
+            .num_img_tokens
+            .iter()
+            .map(|&n| {
+                // Mirrors MiniMaxM3VLMultiModalProcessor._get_prompt_updates:
+                // full = [start_token_id] + [image_token_id] * N + [end_token_id]
+                // The image_token_id positions are marked as embed tokens.
+                let mut tokens = Vec::with_capacity(n + 2);
+                tokens.push(start_token_id);
+                tokens.extend(std::iter::repeat_n(image_token_id, n));
+                tokens.push(end_token_id);
+                PromptReplacement::sequence(Modality::Image, Self::IMAGE_TOKEN, tokens)
+            })
+            .collect())
+    }
+
+    fn field_layouts(&self) -> HashMap<String, FieldLayout> {
+        HashMap::from([
+            (
+                "pixel_values".to_string(),
+                FieldLayout::flat("patches_per_image"),
+            ),
+            ("image_grid_thw".to_string(), FieldLayout::Batched),
+            ("patches_per_image".to_string(), FieldLayout::Batched),
+        ])
+    }
+
+    fn keep_on_cpu_keys(&self) -> Vec<String> {
+        vec!["image_grid_thw".to_string()]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use crate::{
+        registry::{test_helpers::*, ModelMetadata, ModelRegistry},
+        types::ImageSize,
+    };
+
+    #[test]
+    fn minimax_m3_matches_model_type() {
+        let tokenizer = TestTokenizer::new(&[
+            ("]<]image[>[", 200025),
+            ("]<]start of image[>[", 200029),
+            ("]<]end of image[>[", 200030),
+        ]);
+        let config = json!({ "model_type": "minimax_m3_vl" });
+        let metadata = ModelMetadata {
+            model_id: "some-custom-model",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec");
+        assert_eq!(spec.name(), "minimax_m3_vl");
+    }
+
+    #[test]
+    fn minimax_m3_matches_model_id() {
+        let tokenizer = TestTokenizer::new(&[
+            ("]<]image[>[", 200025),
+            ("]<]start of image[>[", 200029),
+            ("]<]end of image[>[", 200030),
+        ]);
+        let config = json!({ "model_type": "minimax_m3_vl" });
+        let metadata = ModelMetadata {
+            model_id: "MiniMaxAI/MiniMax-M3",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec");
+        assert_eq!(spec.name(), "minimax_m3_vl");
+    }
+
+    #[test]
+    fn minimax_m3_prompt_replacements_wrap_with_start_end() {
+        let tokenizer = TestTokenizer::new(&[
+            ("]<]image[>[", 200025),
+            ("]<]start of image[>[", 200029),
+            ("]<]end of image[>[", 200030),
+        ]);
+        let config = json!({ "model_type": "minimax_m3_vl" });
+        let metadata = ModelMetadata {
+            model_id: "MiniMaxAI/MiniMax-M3",
+            tokenizer: &tokenizer,
+            config: &config,
+        };
+        let registry = ModelRegistry::new();
+        let spec = registry.lookup(&metadata).expect("minimax_m3_vl spec");
+
+        // 672x672 image → 576 tokens (48x48 grid, merge_size=2 → 48*48/4=576)
+        let replacements = spec
+            .prompt_replacements(
+                &metadata,
+                &test_preprocessed_with_tokens(&[ImageSize::new(672, 672)], &[576]),
+            )
+            .unwrap();
+
+        assert_eq!(replacements.len(), 1);
+        let tokens = &replacements[0].tokens;
+        assert_eq!(tokens.len(), 578); // 1 start + 576 image + 1 end
+        assert_eq!(tokens[0], 200029); // start
+        assert!(tokens[1..577].iter().all(|&t| t == 200025)); // image tokens
+        assert_eq!(tokens[577], 200030); // end
+    }
+}

src/registry/mod.rs

@@ -1,6 +1,7 @@
 mod kimi_k25;
 mod llama4;
 mod llava;
+mod minimax_m3;
 mod phi3_v;
 mod qwen3_vl;
 mod qwen_vl;
@@ -9,6 +10,7 @@ mod traits;
 use kimi_k25::KimiK25VisionSpec;
 use llama4::Llama4Spec;
 use llava::{LlavaNextSpec, LlavaSpec};
+use minimax_m3::MiniMaxM3VisionSpec;
 use once_cell::sync::Lazy;
 use phi3_v::Phi3VisionSpec;
 use qwen3_vl::Qwen3VLVisionSpec;
@@ -31,6 +33,7 @@ impl ModelRegistry {
                 // LlavaNext must be registered before Llava so "llava_next" model_type matches first.
                 LazySpec::new("llava_next", || Box::new(LlavaNextSpec)),
                 LazySpec::new("llava", || Box::new(LlavaSpec)),
+                LazySpec::new("minimax_m3_vl", || Box::new(MiniMaxM3VisionSpec)),
                 // Qwen3-VL must be registered before QwenVL so "qwen3" matches first.
                 LazySpec::new("qwen3_vl", || Box::new(Qwen3VLVisionSpec)),
                 LazySpec::new("qwen_vl", || Box::new(QwenVLVisionSpec)),

src/vision/image_processor.rs

@@ -474,6 +474,16 @@ impl ImageProcessorRegistry {
             Box::new(super::processors::KimiK25Processor::new()),
         );
 
+        // Register MiniMax-M3 VL
+        registry.register(
+            "minimax-m3-vl",
+            Box::new(super::processors::MiniMaxM3Processor::new()),
+        );
+        registry.register(
+            "minimax_m3_vl",
+            Box::new(super::processors::MiniMaxM3Processor::new()),
+        );
+
         registry
     }
 }

src/vision/mod.rs

@@ -40,7 +40,7 @@ pub use image_processor::{
 };
 pub use preprocessor_config::PreProcessorConfig;
 pub use processors::{
-    Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, Phi3VisionProcessor,
-    Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor,
+    Llama4VisionProcessor, LlavaNextProcessor, LlavaProcessor, MiniMaxM3Processor,
+    Phi3VisionProcessor, Phi4VisionProcessor, PixtralProcessor, Qwen2VLProcessor, Qwen3VLProcessor,
 };
 pub use transforms::TransformError;