From 3401545d29e871255d782a8d09c7a0b53645d6b1 Mon Sep 17 00:00:00 2001 From: Simo Lin <25425177+slin1237@users.noreply.github.com> Date: Wed, 8 Jul 2026 22:30:56 -0700 Subject: [PATCH 1/2] feat(multimodal): vLLM video via shared Modality enum Enable video inputs on the vLLM gRPC path, previously rejected by ensure_image_only. Video pixel tensors ride the same inline/SHM transport as images (#1893), routed to vLLM's video modality. Instead of a one-off is_video bool, hoist the Modality enum into common.proto (like ShmHandle in #1) so the single-modality, precomputed-tensor engines (vLLM + TokenSpeed) share one modality type; TokenSpeed's proto now references smg.grpc.common.Modality. (SGLang keeps its string `modalities` for mixed-modality inputs; converging it onto the common enum is a follow-up.) - proto: Modality enum -> common.proto; tokenspeed references it; vLLM MultimodalInputs gets `common.Modality modality = 10` - assemble: vLLM accepts image or video; assemble_vllm maps the modality and takes mm_hashes from videos vs images. Mixed rejected upstream in process. - servicer: video -> pixel_values_videos under vLLM's `video` MultiModalFieldConfig; tokenspeed servicer reads modality from common_pb2 - test: modality proto round-trip Signed-off-by: Simo Lin <25425177+slin1237@users.noreply.github.com> --- crates/grpc_client/proto/common.proto | 9 ++++ .../proto/tokenspeed_scheduler.proto | 11 +---- crates/grpc_client/proto/vllm_engine.proto | 5 +++ crates/grpc_client/python/pyproject.toml | 2 +- crates/grpc_client/src/lib.rs | 7 ++- grpc_servicer/pyproject.toml | 2 +- .../tokenspeed/encoder_servicer.py | 8 ++-- .../smg_grpc_servicer/tokenspeed/servicer.py | 10 ++--- .../smg_grpc_servicer/vllm/servicer.py | 34 +++++++++----- .../src/routers/grpc/multimodal/assemble.rs | 44 ++++++++++++++++++- .../src/routers/grpc/proto_wrapper.rs | 44 ++++++++++++++++--- 11 files changed, 137 insertions(+), 39 deletions(-) diff --git a/crates/grpc_client/proto/common.proto b/crates/grpc_client/proto/common.proto index c648b858f..1c01886f0 100644 --- a/crates/grpc_client/proto/common.proto +++ b/crates/grpc_client/proto/common.proto @@ -115,3 +115,12 @@ message RemoteTensorHandle { bytes descriptor = 2; uint64 nbytes = 3; } + +// Multimodal input modality, shared across engine protos so the servicers can +// route each request to the matching encoder (image vs video vs audio). +enum Modality { + MODALITY_UNSPECIFIED = 0; + IMAGE = 1; + AUDIO = 2; + VIDEO = 3; +} diff --git a/crates/grpc_client/proto/tokenspeed_scheduler.proto b/crates/grpc_client/proto/tokenspeed_scheduler.proto index 9ba707302..f638a0db2 100644 --- a/crates/grpc_client/proto/tokenspeed_scheduler.proto +++ b/crates/grpc_client/proto/tokenspeed_scheduler.proto @@ -191,15 +191,8 @@ message PlaceholderRange { uint32 length = 2; } -enum Modality { - MODALITY_UNSPECIFIED = 0; - IMAGE = 1; - AUDIO = 2; - VIDEO = 3; -} - message MultimodalItem { - Modality modality = 1; + smg.grpc.common.Modality modality = 1; bytes content_hash = 2; // Primary input to the multimodal encoder for this item. For vision models // this is the preprocessed image/video tensor; for audio models this can be @@ -325,7 +318,7 @@ message GetModelInfoResponse { // non-vision workers before tokenization). bool supports_vision = 14; bool supports_multimodal = 15; - repeated Modality supported_modalities = 16; + repeated smg.grpc.common.Modality supported_modalities = 16; string model_dtype = 17; string multimodal_encoder_dtype = 18; } diff --git a/crates/grpc_client/proto/vllm_engine.proto b/crates/grpc_client/proto/vllm_engine.proto index 2b987dacc..fd637f635 100644 --- a/crates/grpc_client/proto/vllm_engine.proto +++ b/crates/grpc_client/proto/vllm_engine.proto @@ -151,6 +151,11 @@ message MultimodalInputs { // Tensor keys that should remain on CPU (not transferred to GPU). // Maps to vLLM's MultiModalFieldConfig keep_on_cpu flag. repeated string keep_on_cpu_keys = 9; + + // Input modality (image/video). The servicer routes video to vLLM's video + // modality (`pixel_values_videos` / `video_grid_thw`) and expands the video + // placeholder token instead of the image one. Unset (0) is treated as image. + smg.grpc.common.Modality modality = 10; } // ===================== diff --git a/crates/grpc_client/python/pyproject.toml b/crates/grpc_client/python/pyproject.toml index 1a0f7bc50..be09e593d 100644 --- a/crates/grpc_client/python/pyproject.toml +++ b/crates/grpc_client/python/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "smg-grpc-proto" -version = "0.4.12" +version = "0.4.13" description = "SMG gRPC proto definitions for vLLM, TRT-LLM, MLX, TokenSpeed, and SGLang" requires-python = ">=3.10" dependencies = [ diff --git a/crates/grpc_client/src/lib.rs b/crates/grpc_client/src/lib.rs index a79eb5094..f6debf6f8 100644 --- a/crates/grpc_client/src/lib.rs +++ b/crates/grpc_client/src/lib.rs @@ -5,7 +5,12 @@ //! and SGLang scheduler backends. pub mod common_proto { - #![allow(clippy::all, clippy::absolute_paths, unused_qualifications)] + #![allow( + clippy::all, + clippy::absolute_paths, + clippy::trivially_copy_pass_by_ref, + unused_qualifications + )] tonic::include_proto!("smg.grpc.common"); } pub mod abort_on_drop; diff --git a/grpc_servicer/pyproject.toml b/grpc_servicer/pyproject.toml index 9dc2a9432..023e75a6b 100644 --- a/grpc_servicer/pyproject.toml +++ b/grpc_servicer/pyproject.toml @@ -8,7 +8,7 @@ version = "0.6.0" description = "SMG gRPC servicer implementations for LLM inference engines (vLLM, MLX, TokenSpeed, SGLang)" requires-python = ">=3.10" dependencies = [ - "smg-grpc-proto>=0.4.11", + "smg-grpc-proto>=0.4.13", "grpcio>=1.81.1", "grpcio-reflection>=1.81.1", "grpcio-health-checking>=1.81.1", diff --git a/grpc_servicer/smg_grpc_servicer/tokenspeed/encoder_servicer.py b/grpc_servicer/smg_grpc_servicer/tokenspeed/encoder_servicer.py index 29019981c..0122ee37c 100644 --- a/grpc_servicer/smg_grpc_servicer/tokenspeed/encoder_servicer.py +++ b/grpc_servicer/smg_grpc_servicer/tokenspeed/encoder_servicer.py @@ -17,9 +17,9 @@ import grpc from smg_grpc_proto.generated import ( + common_pb2, tokenspeed_encoder_pb2, tokenspeed_encoder_pb2_grpc, - tokenspeed_scheduler_pb2, ) from smg_grpc_servicer.tokenspeed.rdma_pixel import RdmaPixelPuller @@ -145,12 +145,12 @@ def _items_from_proto(self, mm_inputs, bootstrap_room: int = 0): } if item_proto.modality in ( - tokenspeed_scheduler_pb2.IMAGE, - tokenspeed_scheduler_pb2.MODALITY_UNSPECIFIED, + common_pb2.IMAGE, + common_pb2.MODALITY_UNSPECIFIED, ): item_modality = Modality.IMAGE grid_key = "image_grid_thw" - elif item_proto.modality == tokenspeed_scheduler_pb2.VIDEO: + elif item_proto.modality == common_pb2.VIDEO: item_modality = Modality.VIDEO grid_key = "video_grid_thw" else: diff --git a/grpc_servicer/smg_grpc_servicer/tokenspeed/servicer.py b/grpc_servicer/smg_grpc_servicer/tokenspeed/servicer.py index a32ab0667..39204afc2 100644 --- a/grpc_servicer/smg_grpc_servicer/tokenspeed/servicer.py +++ b/grpc_servicer/smg_grpc_servicer/tokenspeed/servicer.py @@ -432,8 +432,8 @@ async def GetModelInfo( self.server_args, "tokenizer_path", "" ) supports_vision = bool(getattr(model_config, "is_multimodal", False)) - image_modality = getattr(tokenspeed_scheduler_pb2, "IMAGE", 1) - video_modality = getattr(tokenspeed_scheduler_pb2, "VIDEO", 3) + image_modality = common_pb2.IMAGE + video_modality = common_pb2.VIDEO supported_modalities = [] if supports_vision: supported_modalities.append(image_modality) @@ -1147,11 +1147,11 @@ def _mm_inputs_from_itemized_proto( @staticmethod def _modality_from_proto(modality: int) -> Modality: - if modality == getattr(tokenspeed_scheduler_pb2, "IMAGE", 1): + if modality == common_pb2.IMAGE: return Modality.IMAGE - if modality == getattr(tokenspeed_scheduler_pb2, "VIDEO", 3): + if modality == common_pb2.VIDEO: return Modality.VIDEO - if modality == getattr(tokenspeed_scheduler_pb2, "AUDIO", 2): + if modality == common_pb2.AUDIO: raise ValueError("TokenSpeed audio multimodal inputs are not supported yet") raise ValueError(f"Unsupported multimodal item modality: {modality}") diff --git a/grpc_servicer/smg_grpc_servicer/vllm/servicer.py b/grpc_servicer/smg_grpc_servicer/vllm/servicer.py index f5778c52d..1b1449518 100755 --- a/grpc_servicer/smg_grpc_servicer/vllm/servicer.py +++ b/grpc_servicer/smg_grpc_servicer/vllm/servicer.py @@ -614,14 +614,26 @@ def _build_preprocessed_mm_inputs( ``batched_keys`` and ``flat_keys`` proto fields. """ prompt_token_ids = list(tokenized.input_ids) - num_images = len(mm_proto.mm_placeholders) + num_items = len(mm_proto.mm_placeholders) + + # Image vs video: vLLM routes each modality to a different encoder and + # expects the pixel tensor under a modality-specific key. The router sends + # the generic ``pixel_values`` field; rename it to ``pixel_values_videos`` + # for the video path (grid/size tensors already carry video-specific keys). + is_video = mm_proto.modality == common_pb2.VIDEO + mm_modality = "video" if is_video else "image" + + def mm_key(key: str) -> str: + if is_video and key == "pixel_values": + return "pixel_values_videos" + return key # Deserialize all tensors from proto hf_dict: dict[str, torch.Tensor] = { - "pixel_values": _tensor_from_proto(mm_proto.pixel_values), + mm_key("pixel_values"): _tensor_from_proto(mm_proto.pixel_values), } for key, td in mm_proto.model_specific_tensors.items(): - hf_dict[key] = _tensor_from_proto(td) + hf_dict[mm_key(key)] = _tensor_from_proto(td) # Cast floating-point tensors to model dtype (e.g. bfloat16). # This mirrors _postprocess_output in multimodal/processing/context.py @@ -631,26 +643,26 @@ def _build_preprocessed_mm_inputs( if hf_dict[key].is_floating_point(): hf_dict[key] = hf_dict[key].to(dtype=model_dtype) - cpu_keys = set(mm_proto.keep_on_cpu_keys) + cpu_keys = {mm_key(k) for k in mm_proto.keep_on_cpu_keys} # Field configs are fully determined by the Rust router. - batched = set(mm_proto.batched_keys) - flat = dict(mm_proto.flat_keys) + batched = {mm_key(k) for k in mm_proto.batched_keys} + flat = {mm_key(k): mm_key(v) for k, v in mm_proto.flat_keys.items()} fields_config: dict[str, MultiModalFieldConfig] = {} flat_sizes_cache: dict[str, torch.Tensor] = {} for key in hf_dict: on_cpu = key in cpu_keys if key in batched: - fields_config[key] = MultiModalFieldConfig.batched("image", keep_on_cpu=on_cpu) + fields_config[key] = MultiModalFieldConfig.batched(mm_modality, keep_on_cpu=on_cpu) elif key in flat: sizes_key = flat[key] if sizes_key not in flat_sizes_cache: flat_sizes_cache[sizes_key] = hf_dict[sizes_key].flatten().to(torch.int64) fields_config[key] = MultiModalFieldConfig.flat_from_sizes( - "image", flat_sizes_cache[sizes_key], keep_on_cpu=on_cpu + mm_modality, flat_sizes_cache[sizes_key], keep_on_cpu=on_cpu ) else: - fields_config[key] = MultiModalFieldConfig.shared("image", num_images) + fields_config[key] = MultiModalFieldConfig.shared(mm_modality, num_items) batch_feature = BatchFeature(hf_dict, tensor_type="pt") mm_kwargs = MultiModalKwargsItems.from_hf_inputs(batch_feature, fields_config) @@ -658,7 +670,7 @@ def _build_preprocessed_mm_inputs( # Build mm_hashes: dict[str, list[str]] mm_hashes: dict[str, list[str]] = {} if mm_proto.mm_hashes: - mm_hashes["image"] = list(mm_proto.mm_hashes) + mm_hashes[mm_modality] = list(mm_proto.mm_hashes) # Build mm_placeholders: dict[str, list[PlaceholderRange]] # When structural tokens (e.g. <|image_start|>, separators) are present @@ -686,7 +698,7 @@ def _build_preprocessed_mm_inputs( placeholders.append( PlaceholderRange(offset=p.offset, length=p.length, is_embed=is_embed) ) - mm_placeholders["image"] = placeholders + mm_placeholders[mm_modality] = placeholders return mm_input( prompt_token_ids=prompt_token_ids, diff --git a/model_gateway/src/routers/grpc/multimodal/assemble.rs b/model_gateway/src/routers/grpc/multimodal/assemble.rs index 50cb5b41f..47bce6b1b 100644 --- a/model_gateway/src/routers/grpc/multimodal/assemble.rs +++ b/model_gateway/src/routers/grpc/multimodal/assemble.rs @@ -11,6 +11,7 @@ use llm_multimodal::{ FieldLayout, Modality, ModelSpecificValue, PlaceholderRange, PreprocessedEncoderInputs, }; use ndarray::ArrayViewD; +use smg_grpc_client::common_proto as common; use tracing::{info, warn}; use super::{ @@ -71,7 +72,7 @@ async fn assemble_multimodal_data_impl( Ok(MultimodalData::Sglang(assemble_sglang(precomputed))) } GrpcClient::Vllm(_) => { - ensure_image_only(&precomputed, "vLLM")?; + ensure_image_or_video(&precomputed, "vLLM")?; Ok(MultimodalData::Vllm(assemble_vllm(precomputed, workers))) } GrpcClient::Trtllm(_) => { @@ -138,6 +139,24 @@ fn ensure_image_only( Ok(()) } +/// Backends that accept both image and video (single-modality per request; +/// mixed image+video is already rejected upstream in `process`). Audio is not +/// supported on these paths. +fn ensure_image_or_video( + intermediate: &PrecomputedMultimodalIntermediate, + backend: &str, +) -> Result<()> { + match intermediate.modality { + // Adds video to the previous image-only gate; ImageEmbeds/Audio stay + // rejected (unchanged from ensure_image_only). + Modality::Image | Modality::Video => Ok(()), + Modality::ImageEmbeds | Modality::Audio => Err(anyhow::anyhow!( + "{backend} multimodal path supports image and video inputs; got {}", + intermediate.modality + )), + } +} + fn assemble_sglang(intermediate: PrecomputedMultimodalIntermediate) -> SglangMultimodalData { let (pixel_values, pixel_values_shape) = serialize_encoder_input(&intermediate.preprocessed); let model_specific_tensors = serialize_model_specific(intermediate.preprocessed.model_specific); @@ -174,7 +193,27 @@ fn assemble_vllm( ) -> VllmMultimodalData { let (pixel_values, pixel_values_shape) = serialize_encoder_input(&intermediate.preprocessed); let model_specific_tensors = serialize_model_specific(intermediate.preprocessed.model_specific); - let mm_hashes = intermediate.images.iter().map(|f| f.hash.clone()).collect(); + let modality = match intermediate.modality { + Modality::Video => common::Modality::Video, + Modality::Audio => common::Modality::Audio, + Modality::Image | Modality::ImageEmbeds => common::Modality::Image, + }; + let is_video = modality == common::Modality::Video; + // Hashes track the per-item media for encoder-output caching: videos for the + // video path, images otherwise. + let mm_hashes = if is_video { + intermediate + .videos + .iter() + .map(|video| video.hash.clone()) + .collect() + } else { + intermediate + .images + .iter() + .map(|frame| frame.hash.clone()) + .collect() + }; let mm_placeholders = intermediate .placeholders .iter() @@ -193,6 +232,7 @@ fn assemble_vllm( batched_keys, flat_keys, keep_on_cpu_keys: intermediate.keep_on_cpu_keys, + modality, shm_enabled: resolve_mm_shm_enabled(workers, false), shm_min_bytes: resolve_mm_shm_min_bytes(workers), } diff --git a/model_gateway/src/routers/grpc/proto_wrapper.rs b/model_gateway/src/routers/grpc/proto_wrapper.rs index 9eb7362f2..c2415f76d 100644 --- a/model_gateway/src/routers/grpc/proto_wrapper.rs +++ b/model_gateway/src/routers/grpc/proto_wrapper.rs @@ -96,6 +96,9 @@ pub struct VllmMultimodalData { pub flat_keys: HashMap, /// Tensor keys that should remain on CPU (`keep_on_cpu=True` in vLLM). pub keep_on_cpu_keys: Vec, + /// Input modality (image/video). Selects the video modality + /// (`pixel_values_videos` / `video_grid_thw`) on the servicer side. + pub modality: common::Modality, /// Resolved per-request SHM transport decision + size threshold (bytes), /// computed upstream (transport mode / worker locality / config). `into_proto` /// uses these to place each tensor inline or in /dev/shm without re-reading @@ -326,6 +329,7 @@ impl VllmMultimodalData { batched_keys: self.batched_keys, flat_keys: self.flat_keys, keep_on_cpu_keys: self.keep_on_cpu_keys, + modality: self.modality as i32, } } } @@ -393,9 +397,9 @@ impl TokenSpeedMultimodalItem { tokenspeed::MultimodalItem { modality: match self.modality { - TokenSpeedModality::Image => tokenspeed::Modality::Image as i32, - TokenSpeedModality::Audio => tokenspeed::Modality::Audio as i32, - TokenSpeedModality::Video => tokenspeed::Modality::Video as i32, + TokenSpeedModality::Image => common::Modality::Image as i32, + TokenSpeedModality::Audio => common::Modality::Audio as i32, + TokenSpeedModality::Video => common::Modality::Video as i32, }, content_hash: self.content_hash, encoder_input, @@ -2042,7 +2046,7 @@ mod tests { assert_eq!(proto.items.len(), 1); let item = &proto.items[0]; - assert_eq!(item.modality, tokenspeed::Modality::Image as i32); + assert_eq!(item.modality, common::Modality::Image as i32); assert_eq!(item.placeholder_token_id, Some(151655)); assert_eq!(item.placeholders[0].offset, 4); assert_eq!(item.placeholders[0].length, 2); @@ -2085,7 +2089,7 @@ mod tests { assert_eq!(proto.items.len(), 1); let item = &proto.items[0]; - assert_eq!(item.modality, tokenspeed::Modality::Video as i32); + assert_eq!(item.modality, common::Modality::Video as i32); assert_eq!(item.placeholder_token_id, Some(151656)); assert_eq!(item.placeholders[0].offset, 4); assert_eq!(item.placeholders[0].length, 2); @@ -2235,4 +2239,34 @@ mod tests { let mut mlx_req = ProtoGenerateRequest::Mlx(Box::default()); mlx_req.set_data_parallel_rank(1); } + + fn vllm_mm_data(modality: common::Modality) -> VllmMultimodalData { + let is_video = modality == common::Modality::Video; + VllmMultimodalData { + pixel_values: vec![0u8; 16], + pixel_values_shape: vec![1, 4], + model_specific_tensors: HashMap::new(), + im_token_id: Some(if is_video { 151656 } else { 151655 }), + mm_placeholders: vec![(3, 4)], + mm_hashes: vec!["h0".to_string()], + batched_keys: vec![], + flat_keys: HashMap::new(), + keep_on_cpu_keys: vec![], + modality, + shm_enabled: false, + shm_min_bytes: 0, + } + } + + #[test] + fn vllm_modality_round_trips_into_proto() { + // The video path must set the proto `modality` so the servicer routes to + // vLLM's video modality; the image path must set image. + let video = vllm_mm_data(common::Modality::Video).into_proto(); + assert_eq!(video.modality, common::Modality::Video as i32); + assert_eq!(video.im_token_id, Some(151656)); + + let image = vllm_mm_data(common::Modality::Image).into_proto(); + assert_eq!(image.modality, common::Modality::Image as i32); + } } From f85e336e2c1b3b52b55e894be94a6aa0e4959ed3 Mon Sep 17 00:00:00 2001 From: Simo Lin <25425177+slin1237@users.noreply.github.com> Date: Thu, 9 Jul 2026 05:50:35 -0700 Subject: [PATCH 2/2] fix(multimodal): correct Qwen3-VL video token layout for vLLM mrope vLLM's Qwen3-VL derives mrope positions by walking one `<|vision_start|>..<|vision_end|>` block per temporal frame (`_iter_mm_grid_hw` loops `for _ in range(t)`), so a video needs `t` per-frame vision blocks and its placeholder range must start on the leading `<|vision_start|>`. Two gaps broke this: - The per-frame video layout was gated to the Qwen3.5 family; base Qwen3-VL fell back to a single flat pad block, so any multi-frame video crashed with "vision_start_token_id not in list" on frame 2. Apply the per-frame layout to the whole family (same processor since #1563) and rename the helper accordingly. - expand_tokens recorded the placeholder range starting *after* the template's leading `<|vision_start|>`, so vLLM's per-frame scan skipped the first marker (breaking even single-frame video). Add `PromptReplacement::structural_prefix` so a spec can fold N preceding template tokens into the reported range; Qwen3-VL video sets it to 1. Verified live on a Qwen3-VL-8B vLLM worker: a red->blue test video now returns "red, blue" over both inline and /dev/shm transport, no crash. Signed-off-by: Simo Lin <25425177+slin1237@users.noreply.github.com> --- crates/multimodal/src/registry/qwen3_vl.rs | 100 ++++++++++++++---- crates/multimodal/src/types.rs | 20 ++++ .../src/routers/grpc/multimodal/process.rs | 39 ++++++- 3 files changed, 134 insertions(+), 25 deletions(-) diff --git a/crates/multimodal/src/registry/qwen3_vl.rs b/crates/multimodal/src/registry/qwen3_vl.rs index 4cbd49c9b..4d832157b 100644 --- a/crates/multimodal/src/registry/qwen3_vl.rs +++ b/crates/multimodal/src/registry/qwen3_vl.rs @@ -55,14 +55,6 @@ impl Qwen3VLVisionSpec { }) } - fn is_qwen3_5(metadata: &ModelMetadata) -> bool { - let id = metadata.model_id.to_ascii_lowercase(); - let model_type = metadata.config_model_type(); - id.contains("qwen3.5") - || id.contains("qwen3.6") - || model_type.is_some_and(|mt| mt == "qwen3_5" || mt == "qwen3_5_moe") - } - fn video_grid_t(preprocessed: &PreprocessedEncoderInputs) -> Option { match preprocessed.model_specific.get("video_grid_thw") { Some(ModelSpecificValue::IntTensor { data, shape }) @@ -92,7 +84,16 @@ impl Qwen3VLVisionSpec { .unwrap_or_default() } - fn qwen3_5_video_replacement_tokens( + /// Build the per-frame video placeholder body for the Qwen3-VL family. + /// + /// Qwen3-VL lays out video as one `<|vision_start|> .. <|vision_end|>` block + /// per temporal frame with a `` timestamp between frames. The chat + /// template already supplies the outer `<|vision_start|>`/`<|vision_end|>`, so + /// this emits only the inner per-frame structure (hence the `grid_idx > 0` + /// guards that reuse the template's opener/closer for the first/last frame). + /// Returns `None` when the layout can't apply (single-frame or ragged token + /// counts), leaving the caller to fall back to a flat pad block. + fn per_frame_video_tokens( metadata: &ModelMetadata, pad_token_id: TokenId, num_tokens: usize, @@ -246,21 +247,29 @@ impl ModelProcessorSpec for Qwen3VLVisionSpec { .feature_token_counts .iter() .map(|&num_tokens| { - let tokens = if Self::is_qwen3_5(metadata) { - video_grid_t - .and_then(|grid_t| { - Self::qwen3_5_video_replacement_tokens( - metadata, - pad_token_id, - num_tokens, - grid_t, - ) - }) - .unwrap_or_else(|| vec![pad_token_id; num_tokens]) - } else { - vec![pad_token_id; num_tokens] - }; + // Every Qwen3-VL model routed to this spec (base VL and the + // 3.5/3.6 family) needs the per-frame video layout: vLLM's + // mrope pass scans for one <|vision_start|> per temporal + // frame, so a single flat block crashes any multi-frame + // video. Fall back to a flat block only when the per-frame + // layout can't be built (single-frame or unknown grid_t). + let tokens = video_grid_t + .and_then(|grid_t| { + Self::per_frame_video_tokens( + metadata, + pad_token_id, + num_tokens, + grid_t, + ) + }) + .unwrap_or_else(|| vec![pad_token_id; num_tokens]); + // The chat template wraps the placeholder as + // <|vision_start|><|video_pad|><|vision_end|>; the leading + // <|vision_start|> belongs to the placeholder range so vLLM's + // per-frame video mrope finds one marker per frame starting at + // the range offset (it scans even for a single frame). PromptReplacement::sequence(Modality::Video, &placeholder_token, tokens) + .with_structural_prefix(1) }) .collect()) } @@ -399,6 +408,51 @@ mod tests { assert!(tokens[82..].iter().all(|&token| token == 151656)); } + #[test] + fn qwen3_vl_video_splits_temporal_grid() { + // Base Qwen3-VL (not the 3.5/3.6 family) must ALSO emit one vision block + // per temporal frame. vLLM's mrope pass scans for a <|vision_start|> per + // frame, so a flat single block crashes any multi-frame video. Regression + // guard for the is_qwen3_5-only gate that previously left base VL flat. + let tokenizer = TestTokenizer::new(&[ + ("<|video_pad|>", 151656), + ("<|vision_start|>", 151652), + ("<|vision_end|>", 151653), + ]); + let config = json!({ + "model_type": "qwen3_vl", + "image_token_id": 151655, + "video_token_id": 151656, + "vision_start_token_id": 151652, + "vision_end_token_id": 151653, + }); + let metadata = ModelMetadata { + model_id: "Qwen/Qwen3-VL-8B-Instruct", + tokenizer: &tokenizer, + config: &config, + }; + let registry = ModelRegistry::new(); + let spec = registry.lookup(&metadata).expect("qwen3_vl spec"); + assert_eq!(spec.name(), "qwen3_vl"); + let preprocessed = test_preprocessed_with_tokens(&[ImageSize::new(320, 256)], &[160]) + .with_extra( + "video_grid_thw", + ModelSpecificValue::int_2d(vec![2, 16, 20], 1, 3), + ); + let replacements = spec + .prompt_replacements_for(&metadata, &preprocessed, crate::types::Modality::Video) + .unwrap(); + + // Two temporal frames -> a vision_end/vision_start seam splits the 160 + // pads into two 80-token halves (mirrors the 3.5 case above). + let tokens = &replacements[0].tokens; + assert_eq!(tokens.len(), 162); + assert!(tokens[..80].iter().all(|&token| token == 151656)); + assert_eq!(tokens[80], 151653); + assert_eq!(tokens[81], 151652); + assert!(tokens[82..].iter().all(|&token| token == 151656)); + } + #[test] fn qwen2_vl_does_not_match_qwen3() { let tokenizer = TestTokenizer::new(&[("", 999)]); diff --git a/crates/multimodal/src/types.rs b/crates/multimodal/src/types.rs index 50aed9b01..47a0b4e1f 100644 --- a/crates/multimodal/src/types.rs +++ b/crates/multimodal/src/types.rs @@ -345,6 +345,15 @@ pub struct PromptReplacement { pub modality: Modality, pub placeholder_token: String, pub tokens: Vec, + /// Number of structural tokens the chat template emits *immediately before* + /// this placeholder (e.g. Qwen's leading `<|vision_start|>`) that belong to + /// the placeholder's range. `expand_tokens` folds them into the reported + /// [`PlaceholderRange`] without re-emitting them, so backends that scan the + /// range for structural markers see the leading marker. vLLM's video mrope + /// walks each frame from `<|vision_start|>` starting at the range offset, so + /// the offset must sit on (or before) the first marker. 0 for the common + /// case where the range is exactly the replacement. + pub structural_prefix: usize, } impl PromptReplacement { @@ -358,6 +367,7 @@ impl PromptReplacement { modality, placeholder_token: placeholder_token.to_string(), tokens: vec![token_id; count], + structural_prefix: 0, } } @@ -366,8 +376,18 @@ impl PromptReplacement { modality, placeholder_token: placeholder_token.to_string(), tokens: sequence, + structural_prefix: 0, } } + + /// Declare that `n` template-emitted structural tokens precede this + /// placeholder and should be included in its reported range. See + /// [`Self::structural_prefix`]. + #[must_use] + pub fn with_structural_prefix(mut self, n: usize) -> Self { + self.structural_prefix = n; + self + } } #[cfg(test)] diff --git a/model_gateway/src/routers/grpc/multimodal/process.rs b/model_gateway/src/routers/grpc/multimodal/process.rs index d9ece9d6f..c45d4756b 100644 --- a/model_gateway/src/routers/grpc/multimodal/process.rs +++ b/model_gateway/src/routers/grpc/multimodal/process.rs @@ -545,9 +545,15 @@ fn expand_tokens( // PromptReplacement uses TokenId = i32, convert to u32 expanded.extend(repl.tokens.iter().map(|&t| t as u32)); + // Fold any template-emitted structural prefix (already in `expanded`, + // e.g. Qwen's leading <|vision_start|>) into the reported range so a + // backend that scans the range for structural markers — vLLM's video + // mrope walks each frame from <|vision_start|> — starts on the marker. + // `offset` (used by the sglang patch_offsets pass above) is untouched. + let prefix = repl.structural_prefix.min(offset); placeholders.push(PlaceholderRange { - offset, - length: repl.tokens.len(), + offset: offset - prefix, + length: repl.tokens.len() + prefix, }); replacement_idx += 1; } else { @@ -593,6 +599,7 @@ mod tests { modality: Modality::Image, placeholder_token: "".to_string(), tokens: vec![50, 50, 50, 50], // Expand to 4 tokens + structural_prefix: 0, }]; let result = expand_tokens(&token_ids, Some(100), None, &replacements); @@ -604,6 +611,30 @@ mod tests { assert!(result.patch_offsets.is_none()); } + #[test] + fn test_expand_tokens_structural_prefix_folds_leading_marker() { + // Qwen video: the chat template emits (777, 100, 778). + // The replacement declares structural_prefix=1, so the reported range must + // start on the leading (offset 2, not 3) and grow by one, letting a + // backend that scans the range for (vLLM's per-frame video mrope) find + // it. The token stream is unchanged — the marker is not re-emitted. + let token_ids = vec![1, 2, 777, 100, 778, 3]; // 100 is the placeholder + let replacements = vec![PromptReplacement { + modality: Modality::Video, + placeholder_token: "<|video_pad|>".to_string(), + tokens: vec![50, 50, 50], // expands to 3 video tokens + structural_prefix: 1, + }]; + + let result = expand_tokens(&token_ids, Some(100), None, &replacements); + + assert_eq!(result.token_ids, vec![1, 2, 777, 50, 50, 50, 778, 3]); + assert_eq!(result.placeholders.len(), 1); + // Range starts on (index 2) and covers it + the 3 video tokens. + assert_eq!(result.placeholders[0].offset, 2); + assert_eq!(result.placeholders[0].length, 4); + } + #[test] fn test_expand_tokens_no_placeholder() { let token_ids = vec![1, 2, 3]; @@ -622,11 +653,13 @@ mod tests { modality: Modality::Image, placeholder_token: "".to_string(), tokens: vec![50, 50], // 2 tokens for first image + structural_prefix: 0, }, PromptReplacement { modality: Modality::Image, placeholder_token: "".to_string(), tokens: vec![60, 60, 60], // 3 tokens for second image + structural_prefix: 0, }, ]; @@ -649,6 +682,7 @@ mod tests { modality: Modality::Image, placeholder_token: "".to_string(), tokens: vec![88, 92, 92, 92, 93, 92, 92, 92, 89], // start + patches + sep + patches + end + structural_prefix: 0, }]; let result = expand_tokens(&token_ids, Some(100), Some(92), &replacements); @@ -674,6 +708,7 @@ mod tests { modality: Modality::Image, placeholder_token: "".to_string(), tokens: vec![50, 50], + structural_prefix: 0, }]; let result = expand_tokens(&token_ids, Some(100), None, &replacements);