Qwen 3.5 lexer support and close the HF normalizer parity gap

Cargo.toml

@@ -58,6 +58,7 @@ once_cell = { version = "1.21.0", default-features = false }
 regex = { version = "1.12.3", default-features = false }
 regex-automata = { version = "0.4", default-features = false }
 unicode-general-category = { version = "1.1.0", default-features = false }
+unicode-normalization = { version = "0.1.25", default-features = false }
 ringbuffer = { version = "0.16", default-features = false }
 strum = { version = "0.27.0", default-features = false, features = ["derive"] }
 thiserror = { version = "2.0.10", default-features = false }
@@ -82,6 +83,7 @@ divan = { package = "codspeed-divan-compat", version = "4.3.0" }
 document-features = "0.2.12"
 humansize = "2.1.3"
 indicatif = "0.18.4"
+indoc = "2.0.7"
 js-sys = "0.3"
 parquet = "58.0.0"
 proptest = "1.10.0"

crates/wordchipper/Cargo.toml

@@ -34,6 +34,7 @@ client = [
     "download",
     "datagym",
     "default-tls",
+    "huggingface",
 ]
 
 ## The download feature enables downloading vocabularies from the internet.
@@ -102,6 +103,12 @@ tracing = [
 testing = []
 
 
+## Enable loading pretrained huggingface modules.
+huggingface = [
+    "std",
+    "dep:tokenizers",
+]
+
 
 [dependencies]
 # macro packages.
@@ -118,6 +125,7 @@ ringbuffer = { workspace = true }
 regex = { workspace = true, features = ["unicode"] }
 regex-automata = { workspace = true, features = ["alloc", "meta", "nfa-thompson", "hybrid", "unicode"] }
 strum = { workspace = true }
+indoc = { workspace = true }
 
 # Provides HashMap/HashSet in no_std mode (non-optional so `default-features = false` just works).
 hashbrown = { workspace = true, features = ["alloc"] }
@@ -131,6 +139,7 @@ serde_json = { workspace = true, optional = true }
 aho-corasick = { workspace = true }
 foldhash = { workspace = true, optional = true }
 unicode-general-category = { workspace = true }
+unicode-normalization = { workspace = true }
 
 # "download" feature deps:
 wordchipper-disk-cache = { version = "0.9.1", path = "../wordchipper-disk-cache", optional = true, default-features = false }
@@ -141,6 +150,8 @@ rayon = { workspace = true, optional = true }
 # "tracing" feature deps:
 tracing = { workspace = true, optional = true }
 
+tokenizers = { workspace = true, features = ["http"], optional = true }
+
 
 [dev-dependencies]
 tempdir = { workspace = true }

crates/wordchipper/README.md

@@ -76,6 +76,11 @@ configuration.
 For a number of pretrained models, simplified constructors are
 available to download, cache, and load the vocabulary.
 
+At this time, we have support for the following loaders:
+
+* `openai:[{PATH}/]{NAME}` - Lod pre-trained OpenAI models.
+* `hf:[{PATH}/]{NAME}` - Load pre-trained HuggingFace models.
+
 See: [wordchipper::get_model](
 https://docs.rs/wordchipper/latest/wordchipper/fn.get_model.html)
 

crates/wordchipper/src/encoders/token_span_encoder/token_span_encoder.rs

@@ -108,6 +108,9 @@ impl<T: TokenType> TokenEncoder<T> for TokenSpanEncoder<T> {
             }
         }
 
+        let normalized_text = self.vocab.normalize_text(text);
+        let text = normalized_text.as_ref();
+
         self.spanner
             .for_each_split_span(text, special_filter, &mut |span_ref| {
                 se.encode_append_span_ref(&self.vocab, text, span_ref, tokens);

crates/wordchipper/src/errors.rs

@@ -5,12 +5,16 @@ use crate::alloc::string::String;
 /// Errors from wordchipper operations.
 #[derive(Debug, thiserror::Error)]
 pub enum WCError {
+    /// Not Implemented Error.
+    #[error("Not Implemented: {0}")]
+    NotImplemented(String),
+
     /// Resource not found.
-    #[error("{0}")]
+    #[error("Resource Not Found: {0}")]
     ResourceNotFound(String),
 
     /// The resource is a duplicate.
-    #[error("{0}")]
+    #[error("Duplicate: {0}")]
     DuplicatedResource(String),
 
     /// Vocab size exceeds the capacity of the target token type.
@@ -28,7 +32,7 @@ pub enum WCError {
     },
 
     /// Vocabulary data is inconsistent.
-    #[error("{0}")]
+    #[error("Vocab Conflict: {0}")]
     VocabConflict(String),
 
     /// Token value out of range for the target type.

crates/wordchipper/src/pretrained/factory/vocab_description.rs

@@ -21,20 +21,24 @@ pub struct VocabDescription {
 
 impl VocabDescription {
     /// Build a new vocabulary description.
-    pub fn new<Q>(
+    pub fn new<Q, C, D>(
         id: Q,
-        context: &[&str],
-        description: &str,
+        context: &[C],
+        description: D,
     ) -> Self
     where
         Q: Into<VocabQuery>,
+        C: AsRef<str>,
+        D: AsRef<str>,
     {
         let id = id.into();
+        let context = context.iter().map(|c| c.as_ref().to_string()).collect();
+        let description = description.as_ref().to_string();
 
         Self {
             id,
-            context: context.iter().map(|&s| s.to_string()).collect(),
-            description: description.to_string(),
+            context,
+            description,
         }
     }
 

crates/wordchipper/src/pretrained/factory/vocab_query.rs

@@ -66,7 +66,7 @@ impl Display for VocabQuery {
 }
 
 impl VocabQuery {
-    /// Build a new query from structure.
+    /// Build a new query.
     pub fn new(
         schema: Option<&str>,
         path: Option<&str>,
@@ -174,13 +174,27 @@ impl VocabQuery {
         }
         query.name() == self.name()
     }
+
+    /// Build a cache context for this query.
+    pub fn to_context(&self) -> Vec<String> {
+        let mut context = Vec::new();
+        if let Some(schema) = self.schema() {
+            context.push(schema.to_string());
+        }
+        if let Some(path) = self.path() {
+            context.extend(path.split('/').map(|p| p.to_string()));
+        }
+        context.push(self.name().to_string());
+        context
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use core::str::FromStr;
 
     use crate::{
+        alloc::vec,
         prelude::*,
         pretrained::factory::vocab_query::VocabQuery,
     };
@@ -202,6 +216,19 @@ mod tests {
             VocabQuery::new(Some("xyz"), Some("foo/bar"), "vocab_name")
         );
     }
+
+    #[test]
+    fn test_to_context() {
+        let q = VocabQuery::from_str("vocab_name").unwrap();
+        assert_eq!(q.to_context(), vec!["vocab_name"]);
+
+        let q = VocabQuery::from_str("foo/bar/vocab_name").unwrap();
+        assert_eq!(q.to_context(), vec!["foo", "bar", "vocab_name"]);
+
+        let q = VocabQuery::from_str("xyz:foo/bar/vocab_name").unwrap();
+        assert_eq!(q.to_context(), vec!["xyz", "foo", "bar", "vocab_name"]);
+    }
+
     #[test]
     fn test_vocab_query_with_schema() {
         let query = VocabQuery::new(None, None, "vocab_name").with_schema(Some("provider"));