test it

rendezqueue · Jul 27, 2024 · 18c7820 · 18c7820
1 parent c514bfd
commit 18c7820
Showing 1 changed file with 17 additions and 0 deletions.
diff --git a/test/chat/opt_test.cc b/test/chat/opt_test.cc
@@ -70,9 +70,26 @@ sentence_terminals_parse_test()
   assert(opt.sentence_terminals.size() == 3);
 }
 
+static
+  void
+substitution_parse_test()
+{
+  rendezllama::ChatOptions opt;
+  FildeshX in[1];
+  *in = FildeshX_of_strlit(
+      "(substitution (special_tokens (()) (() (name <|im_start|>))))");
+  bool all_good = slurp_sxpb_initialize_options_close_FildeshX( in, opt, "");
+  assert(all_good);
+  assert(opt.special_tokens.size() == 1);
+  assert(opt.special_tokens[0].candidates.size() == 1);
+  assert(opt.special_tokens[0].alias == "<|im_start|>");
+  assert(opt.special_tokens[0].candidates[0] == "<|im_start|>");
+}
+
 int main()
 {
   chat_prefixes_parse_test();
   sentence_terminals_parse_test();
+  substitution_parse_test();
   return 0;
 }