@@ -20,11 +20,114 @@ use async_trait::async_trait;
2020/// provider. Drift between the two silently splits one embedding space into
2121/// two, and every vector written on the wrong side of the split becomes
2222/// unsearchable without a re-embed.
23+ /// # Delimiters in a component
24+ ///
25+ /// A component containing `;`, `=` or `%` is percent-encoded, because without
26+ /// that the format is ambiguous: `("a;model=b", "c")` and `("a", "b;model=c")`
27+ /// are different embedding spaces that would otherwise produce one identical
28+ /// key, and vectors from both would then be compared as though they came from
29+ /// the same model.
30+ ///
31+ /// Encoding only those three characters is what keeps this from being a
32+ /// migration. Every provider and model identifier actually in use is
33+ /// alphanumeric plus `-`, `_`, `.`, `/` or `:`, and each of those passes
34+ /// through untouched — so every signature already on disk still formats to the
35+ /// same bytes. Only a name that could have collided changes, and such a name
36+ /// has never been written.
2337#[ must_use]
2438pub fn format_embedding_signature ( name : & str , model_id : & str , dims : usize ) -> String {
39+ let name = escape_component ( name) ;
40+ let model_id = escape_component ( model_id) ;
2541 format ! ( "provider={name};model={model_id};dims={dims}" )
2642}
2743
44+ /// Percent-encode the three characters that carry structure in a signature.
45+ ///
46+ /// `%` goes first and must: encoding it afterwards would re-encode the `%` this
47+ /// function just introduced, and `a;b` would arrive as `a%3Bb` from one path
48+ /// and `a%253Bb` from another.
49+ fn escape_component ( value : & str ) -> String {
50+ if !value. contains ( [ '%' , ';' , '=' ] ) {
51+ // The overwhelmingly common path, and the one that guarantees existing
52+ // keys are untouched: no allocation beyond the copy, no rewriting.
53+ return value. to_string ( ) ;
54+ }
55+ value
56+ . replace ( '%' , "%25" )
57+ . replace ( ';' , "%3B" )
58+ . replace ( '=' , "%3D" )
59+ }
60+
61+ #[ cfg( test) ]
62+ mod embedding_signature_tests {
63+ use super :: format_embedding_signature;
64+
65+ /// The signature format is a **persisted key**, pinned to literal values.
66+ ///
67+ /// Written against golden strings rather than against another copy of the
68+ /// function on purpose: the host used to hold a byte-identical duplicate of
69+ /// this file and the two silently diverged once already. A guard that
70+ /// compares two implementations stops protecting anything the moment one of
71+ /// them goes away — which is exactly what happened when the duplicate was
72+ /// removed. Literals outlive that.
73+ ///
74+ /// Every vector on disk is keyed by one of these strings, so a change here
75+ /// is a migration, never an edit.
76+ #[ test]
77+ fn signature_format_is_pinned_to_its_persisted_form ( ) {
78+ assert_eq ! (
79+ format_embedding_signature( "ollama" , "nomic-embed-text" , 768 ) ,
80+ "provider=ollama;model=nomic-embed-text;dims=768"
81+ ) ;
82+ assert_eq ! (
83+ format_embedding_signature( "none" , "none" , 0 ) ,
84+ "provider=none;model=none;dims=0"
85+ ) ;
86+ }
87+
88+ /// Two distinct embedding spaces must never share one signature.
89+ ///
90+ /// Without escaping these two collide exactly: both format to
91+ /// `provider=a;model=b;model=c;dims=3`. A collision here is not a cosmetic
92+ /// problem — the signature is what decides which vectors are comparable, so
93+ /// two models' vectors would be scored against each other as though they
94+ /// came from one space.
95+ #[ test]
96+ fn delimiter_characters_cannot_make_distinct_spaces_collide ( ) {
97+ let first = format_embedding_signature ( "a;model=b" , "c" , 3 ) ;
98+ let second = format_embedding_signature ( "a" , "b;model=c" , 3 ) ;
99+ assert_ne ! ( first, second) ;
100+ }
101+
102+ /// Escaping `%` last would make the encoding itself ambiguous.
103+ #[ test]
104+ fn an_already_percent_encoded_name_does_not_collide_with_a_literal_one ( ) {
105+ assert_ne ! (
106+ format_embedding_signature( "a%3Bb" , "m" , 3 ) ,
107+ format_embedding_signature( "a;b" , "m" , 3 )
108+ ) ;
109+ }
110+
111+ /// The escaping is not a migration: every identifier shaped like the ones
112+ /// actually in use formats to the same bytes it always did.
113+ #[ test]
114+ fn identifiers_in_real_use_are_untouched_by_the_escaping ( ) {
115+ for ( provider, model) in [
116+ ( "ollama" , "nomic-embed-text" ) ,
117+ ( "openai" , "text-embedding-3-small" ) ,
118+ ( "huggingface" , "sentence-transformers/all-MiniLM-L6-v2" ) ,
119+ ( "local" , "bge_base.en-v1.5" ) ,
120+ ( "backend" , "tinyhumans:default" ) ,
121+ ] {
122+ assert_eq ! (
123+ format_embedding_signature( provider, model, 768 ) ,
124+ format!( "provider={provider};model={model};dims=768" ) ,
125+ "{provider}/{model} must not be rewritten — it is a persisted key"
126+ ) ;
127+ }
128+ }
129+ }
130+
28131/// Converts text into numerical vectors.
29132#[ async_trait]
30133pub trait EmbeddingProvider : Send + Sync {
0 commit comments