From e4fbe1ca0f750860326ce15528b85f2cdc1a8142 Mon Sep 17 00:00:00 2001
From: grencez <git@grencez.dev>
Date: Sat, 27 Jul 2024 16:11:16 -0700
Subject: [PATCH] more test

---
 src/chat/opt.cc       | 10 +---------
 src/chat/opt.hh       |  7 +++++++
 test/chat/opt_test.cc | 23 ++++++++++++++++++++++-
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/src/chat/opt.cc b/src/chat/opt.cc
index 1d0193f..9a5c30a 100644
--- a/src/chat/opt.cc
+++ b/src/chat/opt.cc
@@ -208,14 +208,6 @@ static int initialize_options(ChatOptions& opt) {
   return exstatus;
 }
 
-static
-  bool
-slurp_sxpb_options_close_FildeshX(
-    FildeshX* in,
-    ChatOptions& opt,
-    const FildeshSxprotoField* schema,
-    const std::string& filename);
-
 static
   bool
 parse_sxpb_file_options(ChatOptions& opt, const char* filename)
@@ -411,7 +403,7 @@ lone_subfield_at_FildeshSxpb_to_cc_string(
 }
 
   bool
-slurp_sxpb_options_close_FildeshX(
+rendezllama::slurp_sxpb_options_close_FildeshX(
     FildeshX* in,
     ChatOptions& opt,
     const FildeshSxprotoField* schema,
diff --git a/src/chat/opt.hh b/src/chat/opt.hh
index 3ba0080..c9c5cfc 100644
--- a/src/chat/opt.hh
+++ b/src/chat/opt.hh
@@ -85,6 +85,12 @@ print_options(std::ostream& out, const ChatOptions& opt);
 int
 parse_options(ChatOptions& opt, int argc, char** argv);
 bool
+slurp_sxpb_options_close_FildeshX(
+    FildeshX* in,
+    rendezllama::ChatOptions& opt,
+    const FildeshSxprotoField* schema,
+    const std::string& filename);
+bool
 slurp_sxpb_initialize_options_close_FildeshX(
     FildeshX* in,
     rendezllama::ChatOptions& opt,
@@ -94,5 +100,6 @@ slurp_sxpb_dynamic_options_close_FildeshX(
     FildeshX* in,
     rendezllama::ChatOptions& opt);
 
+
 }  // namespace rendezllama
 #endif
diff --git a/test/chat/opt_test.cc b/test/chat/opt_test.cc
index cb57469..404d4ea 100644
--- a/test/chat/opt_test.cc
+++ b/test/chat/opt_test.cc
@@ -78,12 +78,33 @@ substitution_parse_test()
   FildeshX in[1];
   *in = FildeshX_of_strlit(
       "(substitution (special_tokens (()) (() (name <|im_start|>))))");
-  bool all_good = slurp_sxpb_initialize_options_close_FildeshX( in, opt, "");
+  bool all_good = rendezllama::slurp_sxpb_options_close_FildeshX(
+      in, opt, rendezllama::options_sxproto_schema(), "");
   assert(all_good);
   assert(opt.special_tokens.size() == 1);
   assert(opt.special_tokens[0].candidates.size() == 1);
   assert(opt.special_tokens[0].alias == "<|im_start|>");
   assert(opt.special_tokens[0].candidates[0] == "<|im_start|>");
+
+  opt.special_tokens.clear();
+  *in = FildeshX_of_strlit(
+      "(substitution\n"
+      " (bos_token_alias <bos_token>)\n"
+      " (eos_token_alias <eos_token>)\n"
+      " (special_tokens (())\n"
+      "  (() (alias <|im_start|>) (candidates (()) <bos_token>))\n"
+      "  (() (alias <|im_end|>) (candidates (()) <eos_token>))\n"
+      "))");
+  all_good = rendezllama::slurp_sxpb_options_close_FildeshX(
+      in, opt, rendezllama::options_sxproto_schema(), "");
+  assert(all_good);
+  assert(opt.special_tokens.size() == 2);
+  assert(opt.special_tokens[0].candidates.size() == 1);
+  assert(opt.special_tokens[1].candidates.size() == 1);
+  assert(opt.special_tokens[0].alias == "<|im_start|>");
+  assert(opt.special_tokens[1].alias == "<|im_end|>");
+  assert(opt.special_tokens[0].candidates[0] == "<bos_token>");
+  assert(opt.special_tokens[1].candidates[0] == "<eos_token>");
 }
 
 int main()