uh?

rendezqueue · Jul 27, 2024 · 1860a45 · 1860a45
1 parent 312758c
commit 1860a45
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/src/chat/opt.cc b/src/chat/opt.cc
@@ -506,7 +506,7 @@ rendezllama::slurp_sxpb_options_close_FildeshX(
     if (!nullish_FildeshSxpbIT(sub_it)) {
       for (sub_it = first_at_FildeshSxpb(sxpb, sub_it); !nullish_FildeshSxpbIT(sub_it);
            sub_it = next_at_FildeshSxpb(sxpb, sub_it)) {
-        auto& special =  opt.special_tokens.emplace_back();
+        rendezllama::SpecialToken special;
         lone_subfield_at_FildeshSxpb_to_cc_string(&special.alias, sxpb, sub_it, "alias");
         assert(!special.alias.empty());
         FildeshSxpbIT candidate_it = lookup_subfield_at_FildeshSxpb(sxpb, sub_it, "candidates");
@@ -520,6 +520,7 @@ rendezllama::slurp_sxpb_options_close_FildeshX(
             special.candidates.push_back(str_value_at_FildeshSxpb(sxpb, candidate_it));
           }
         }
+        opt.special_tokens.push_back(special);
       }
     }
   }

diff --git a/src/language/vocabulary.cc b/src/language/vocabulary.cc
@@ -120,7 +120,7 @@ Vocabulary::tokenize_special(std::string_view s) const
       /*add_bos=*/false,
       /*special=*/true);
   if (n != 1) {
-    token_id = this->cardinality();
+    token_id = Vocabulary::null_token_id;
   }
   return token_id;
 }