diff --git a/crates/multimodal/src/registry/inkling.rs b/crates/multimodal/src/registry/inkling.rs index 3b55b2a5f..45bb93df1 100644 --- a/crates/multimodal/src/registry/inkling.rs +++ b/crates/multimodal/src/registry/inkling.rs @@ -15,16 +15,15 @@ use crate::{ pub(super) struct InklingSpec; impl InklingSpec { - const CONTENT_IMAGE: &'static str = "<|content_image|>"; - const CONTENT_AUDIO_INPUT: &'static str = "<|content_audio_input|>"; - const AUDIO: &'static str = "<|audio|>"; + const IMAGE_PLACEHOLDER: &'static str = "<|unused_200054|>"; + const AUDIO_PLACEHOLDER: &'static str = "<|unused_200053|>"; - fn image_transport_fill_id(metadata: &ModelMetadata) -> RegistryResult { - metadata.token_id(Self::CONTENT_IMAGE) + fn image_placeholder_id(metadata: &ModelMetadata) -> RegistryResult { + metadata.token_id(Self::IMAGE_PLACEHOLDER) } fn audio_placeholder_id(metadata: &ModelMetadata) -> RegistryResult { - metadata.token_id(Self::AUDIO) + metadata.token_id(Self::AUDIO_PLACEHOLDER) } fn tower_enabled(metadata: &ModelMetadata, config_key: &str) -> bool { @@ -54,11 +53,11 @@ impl ModelProcessorSpec for InklingSpec { } fn placeholder_token(&self, _metadata: &ModelMetadata) -> RegistryResult { - Ok(Self::CONTENT_IMAGE.to_string()) + Ok(Self::IMAGE_PLACEHOLDER.to_string()) } fn placeholder_token_id(&self, metadata: &ModelMetadata) -> RegistryResult { - Self::image_transport_fill_id(metadata) + Self::image_placeholder_id(metadata) } fn placeholder_token_for( @@ -67,8 +66,8 @@ impl ModelProcessorSpec for InklingSpec { modality: Modality, ) -> RegistryResult { match modality { - Modality::Image => Ok(Self::CONTENT_IMAGE.to_string()), - Modality::Audio => Ok(Self::CONTENT_AUDIO_INPUT.to_string()), + Modality::Image => Ok(Self::IMAGE_PLACEHOLDER.to_string()), + Modality::Audio => Ok(Self::AUDIO_PLACEHOLDER.to_string()), _ => Err(ModelRegistryError::UnsupportedModality { spec: self.name(), modality, @@ -82,7 +81,7 @@ impl ModelProcessorSpec for InklingSpec { modality: Modality, ) -> RegistryResult { match modality { - Modality::Image => Self::image_transport_fill_id(metadata), + Modality::Image => Self::image_placeholder_id(metadata), Modality::Audio => Self::audio_placeholder_id(metadata), _ => Err(ModelRegistryError::UnsupportedModality { spec: self.name(), @@ -135,40 +134,42 @@ impl ModelProcessorSpec for InklingSpec { ) -> RegistryResult> { match modality { Modality::Image => { - let content_image_id = metadata.token_id(Self::CONTENT_IMAGE)?; + let image_placeholder_id = Self::image_placeholder_id(metadata)?; Ok(preprocessed .feature_token_counts .iter() .map(|&num_tokens| { - let mut tokens = Vec::with_capacity(num_tokens + 1); - tokens.push(content_image_id); - // TML transports images as a typed span and does not - // define a positive image target token. TokenSpeed only - // needs real ids until it rewrites the explicit feature - // offsets to content-derived MM pad values, so reuse the - // public content token as an internal transport fill. - tokens.extend(std::iter::repeat_n(content_image_id, num_tokens)); - PromptReplacement::sequence(Modality::Image, Self::CONTENT_IMAGE, tokens) - .with_feature_span(1, num_tokens) + let tokens = vec![image_placeholder_id; num_tokens]; + PromptReplacement::sequence( + Modality::Image, + Self::IMAGE_PLACEHOLDER, + tokens, + ) + .with_feature_span(0, num_tokens) + // The checkpoint-provided template emits + // `<|content_image|>` immediately before the one soft + // placeholder that this replacement expands. + .with_structural_prefix(1) }) .collect()) } Modality::Audio => { - let content_audio_id = metadata.token_id(Self::CONTENT_AUDIO_INPUT)?; - let audio_id = Self::audio_placeholder_id(metadata)?; + let audio_placeholder_id = Self::audio_placeholder_id(metadata)?; Ok(preprocessed .feature_token_counts .iter() .map(|&num_tokens| { - let mut tokens = Vec::with_capacity(num_tokens + 1); - tokens.push(content_audio_id); - tokens.extend(std::iter::repeat_n(audio_id, num_tokens)); + let tokens = vec![audio_placeholder_id; num_tokens]; PromptReplacement::sequence( Modality::Audio, - Self::CONTENT_AUDIO_INPUT, + Self::AUDIO_PLACEHOLDER, tokens, ) - .with_feature_span(1, num_tokens) + .with_feature_span(0, num_tokens) + // The checkpoint-provided template keeps the typed + // audio marker before the soft placeholder and owns + // `<|audio_end|>`. + .with_structural_prefix(1) }) .collect()) } @@ -206,7 +207,8 @@ mod tests { TestTokenizer::new(&[ ("<|content_image|>", 200005), ("<|content_audio_input|>", 200020), - ("<|audio|>", 200023), + ("<|unused_200054|>", 200054), + ("<|unused_200053|>", 200053), ]) } @@ -324,7 +326,7 @@ mod tests { } #[test] - fn image_replacement_preserves_content_token_and_adds_patch_span() { + fn image_replacement_expands_checkpoint_soft_placeholder_after_content_marker() { let tokenizer = tokenizer(); let config = config(); let metadata = ModelMetadata { @@ -342,23 +344,30 @@ mod tests { crate::types::Modality::Image, ) .unwrap(); - assert_eq!(replacements[0].tokens, vec![200005, 200005, 200005, 200005]); + assert_eq!(replacements[0].placeholder_token, "<|unused_200054|>"); + assert_eq!(replacements[0].tokens, vec![200054, 200054, 200054]); assert_eq!( replacements[0].feature_ranges, Some(vec![crate::types::PlaceholderRange { - offset: 1, + offset: 0, length: 3 }]) ); + assert_eq!(replacements[0].structural_prefix, 1); + assert_eq!( + spec.placeholder_token_for(&metadata, crate::types::Modality::Image) + .unwrap(), + "<|unused_200054|>" + ); assert_eq!( spec.placeholder_token_id_for(&metadata, crate::types::Modality::Image) .unwrap(), - 200005 + 200054 ); } #[test] - fn audio_replacement_preserves_content_token_and_adds_placeholder_span() { + fn audio_replacement_expands_checkpoint_soft_placeholder_after_content_marker() { let tokenizer = tokenizer(); let config = config(); let metadata = ModelMetadata { @@ -376,18 +385,25 @@ mod tests { crate::types::Modality::Audio, ) .unwrap(); - assert_eq!(replacements[0].tokens, vec![200020, 200023, 200023]); + assert_eq!(replacements[0].placeholder_token, "<|unused_200053|>"); + assert_eq!(replacements[0].tokens, vec![200053, 200053]); assert_eq!( replacements[0].feature_ranges, Some(vec![crate::types::PlaceholderRange { - offset: 1, + offset: 0, length: 2 }]) ); + assert_eq!(replacements[0].structural_prefix, 1); + assert_eq!( + spec.placeholder_token_for(&metadata, crate::types::Modality::Audio) + .unwrap(), + "<|unused_200053|>" + ); assert_eq!( spec.placeholder_token_id_for(&metadata, crate::types::Modality::Audio) .unwrap(), - 200023 + 200053 ); } } diff --git a/crates/tokenizer/src/chat_template.rs b/crates/tokenizer/src/chat_template.rs index 241e625ca..ef0cc9aab 100644 --- a/crates/tokenizer/src/chat_template.rs +++ b/crates/tokenizer/src/chat_template.rs @@ -832,6 +832,12 @@ fn sort_json_keys(value: &JsonValue) -> JsonValue { } } +/// Hugging Face chat-template helper for surfacing model-authored validation +/// errors instead of a generic "unknown function" render failure. +fn raise_exception(message: String) -> std::result::Result { + Err(MinijinjaError::new(ErrorKind::InvalidOperation, message)) +} + /// Build a pre-configured `Environment<'static>` with the given template string, /// Python-compat method callback, and custom `tojson` filter already registered. /// The template is stored under the name `"chat"` using owned storage so the @@ -856,6 +862,7 @@ fn build_environment(template: String) -> Result> { // This overrides minijinja's built-in tojson to support additional kwargs // like ensure_ascii, separators, and sort_keys that HuggingFace templates use env.add_filter("tojson", tojson_filter); + env.add_function("raise_exception", raise_exception); Ok(env) } @@ -1169,6 +1176,21 @@ mod tests { assert!(result.is_err()); } + #[test] + fn test_raise_exception_surfaces_template_validation_message() { + let state = ChatTemplateState::new(Some( + "{{ raise_exception('reasoning_effort is invalid') }}".to_string(), + )) + .unwrap(); + + let error = state + .apply(&[], ChatTemplateParams::default()) + .unwrap_err() + .to_string(); + + assert!(error.contains("reasoning_effort is invalid"), "{error}"); + } + #[test] fn test_special_tokens_injected_into_context() { let template = "{{ bos_token }}{% for message in messages %}{{ message.content }}{% endfor %}{{ eos_token }}";