more test

src/chat/opt.cc

@@ -208,14 +208,6 @@ static int initialize_options(ChatOptions& opt) {
   return exstatus;
 }
 
-static
-  bool
-slurp_sxpb_options_close_FildeshX(
-    FildeshX* in,
-    ChatOptions& opt,
-    const FildeshSxprotoField* schema,
-    const std::string& filename);
-
 static
   bool
 parse_sxpb_file_options(ChatOptions& opt, const char* filename)
@@ -411,7 +403,7 @@ lone_subfield_at_FildeshSxpb_to_cc_string(
 }
 
   bool
-slurp_sxpb_options_close_FildeshX(
+rendezllama::slurp_sxpb_options_close_FildeshX(
     FildeshX* in,
     ChatOptions& opt,
     const FildeshSxprotoField* schema,

src/chat/opt.hh

@@ -85,6 +85,12 @@ print_options(std::ostream& out, const ChatOptions& opt);
 int
 parse_options(ChatOptions& opt, int argc, char** argv);
 bool
+slurp_sxpb_options_close_FildeshX(
+    FildeshX* in,
+    rendezllama::ChatOptions& opt,
+    const FildeshSxprotoField* schema,
+    const std::string& filename);
+bool
 slurp_sxpb_initialize_options_close_FildeshX(
     FildeshX* in,
     rendezllama::ChatOptions& opt,
@@ -94,5 +100,6 @@ slurp_sxpb_dynamic_options_close_FildeshX(
     FildeshX* in,
     rendezllama::ChatOptions& opt);
 
+
 }  // namespace rendezllama
 #endif

test/chat/opt_test.cc

@@ -78,12 +78,33 @@ substitution_parse_test()
   FildeshX in[1];
   *in = FildeshX_of_strlit(
       "(substitution (special_tokens (()) (() (name <|im_start|>))))");
-  bool all_good = slurp_sxpb_initialize_options_close_FildeshX( in, opt, "");
+  bool all_good = rendezllama::slurp_sxpb_options_close_FildeshX(
+      in, opt, rendezllama::options_sxproto_schema(), "");
   assert(all_good);
   assert(opt.special_tokens.size() == 1);
   assert(opt.special_tokens[0].candidates.size() == 1);
   assert(opt.special_tokens[0].alias == "<|im_start|>");
   assert(opt.special_tokens[0].candidates[0] == "<|im_start|>");
+
+  opt.special_tokens.clear();
+  *in = FildeshX_of_strlit(
+      "(substitution\n"
+      " (bos_token_alias <bos_token>)\n"
+      " (eos_token_alias <eos_token>)\n"
+      " (special_tokens (())\n"
+      "  (() (alias <|im_start|>) (candidates (()) <bos_token>))\n"
+      "  (() (alias <|im_end|>) (candidates (()) <eos_token>))\n"
+      "))");
+  all_good = rendezllama::slurp_sxpb_options_close_FildeshX(
+      in, opt, rendezllama::options_sxproto_schema(), "");
+  assert(all_good);
+  assert(opt.special_tokens.size() == 2);
+  assert(opt.special_tokens[0].candidates.size() == 1);
+  assert(opt.special_tokens[1].candidates.size() == 1);
+  assert(opt.special_tokens[0].alias == "<|im_start|>");
+  assert(opt.special_tokens[1].alias == "<|im_end|>");
+  assert(opt.special_tokens[0].candidates[0] == "<bos_token>");
+  assert(opt.special_tokens[1].candidates[0] == "<eos_token>");
 }
 
 int main()