update tokenizer to bpe.

wangzhaode · Sep 26, 2023 · f0f8a90 · f0f8a90
1 parent 412b6fa
commit f0f8a90
Show file tree

Hide file tree

Showing 61 changed files with 518 additions and 1,183,467 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.0)
 project(mnn-llm)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
 
 option(BUILD_FOR_ANDROID "Build for android whith mini memory mode." OFF)
 option(WITH_CUDA "Enable CUDA support" OFF)

diff --git a/android/app/src/main/java/com/mnn/llm/Chat.java b/android/app/src/main/java/com/mnn/llm/Chat.java
@@ -3,7 +3,7 @@
 import java.io.Serializable;
 
 public class Chat implements Serializable {
-    public native boolean Init(String modelDir, String tokenizerDir);
+    public native boolean Init(String modelDir);
     public native boolean Ready();
     public native float Progress();
     public native String Submit(String input);

diff --git a/android/app/src/main/java/com/mnn/llm/MainActivity.java b/android/app/src/main/java/com/mnn/llm/MainActivity.java
@@ -29,7 +29,6 @@ public class MainActivity extends AppCompatActivity {
     private TextView mProcessPercent;
     // resource files
     private String mModelDir = "/data/local/tmp/model";
-    private String mTokenizerDir = "";
     private boolean mModelNeedDownload = true;
     @Override
     protected void onCreate(Bundle savedInstanceState) {
@@ -116,7 +115,6 @@ public void loadModel(View view) {
         mLoadButton.setText("模型加载中 ...");
         mProcessView.setVisibility(View.VISIBLE);
         mChat = new Chat();
-        prepareFiles();
         System.out.println("[MNN_DEBUG] is chat Ready: " + mChat.Ready());
         Handler handler = new Handler() {
             @Override
@@ -126,37 +124,25 @@ public void handleMessage(Message msg) {
             }
         };
         // copy models
-        LoadThread loadT = new LoadThread(mChat, handler, mModelDir, mTokenizerDir);
+        LoadThread loadT = new LoadThread(mChat, handler, mModelDir);
         loadT.start();
         ProgressThread progressT = new ProgressThread(mChat, mProcessHandler);
         progressT.start();
     }
-
-    public void prepareFiles() {
-        System.out.println("MNN_DEBUG: prepareFiles Start");
-        try {
-            mTokenizerDir = Common.copyAssetResource2File(this, "tokenizer");
-        } catch (Exception e) {
-            System.out.println(e.toString());
-        }
-        System.out.println("MNN_DEBUG: prepareFiles End" + mModelDir + " # " + mTokenizerDir);
-    }
 }
 
 class LoadThread extends Thread {
     private Chat mChat;
     private Handler mHandler;
     private String mModelDir;
-    private String mTokenizerDir;
-    LoadThread(Chat chat, Handler handler, String modelDir, String tokenizerDir) {
+    LoadThread(Chat chat, Handler handler, String modelDir) {
         mChat = chat;
         mHandler = handler;
         mModelDir = modelDir;
-        mTokenizerDir = tokenizerDir;
     }
     public void run() {
         super.run();
-        mChat.Init(mModelDir, mTokenizerDir);
+        mChat.Init(mModelDir);
         mHandler.sendMessage(new Message());
     }
 }

diff --git a/android/app/src/main/jni/CMakeLists.txt b/android/app/src/main/jni/CMakeLists.txt
@@ -7,6 +7,7 @@ cmake_minimum_required(VERSION 3.10)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/../../../../../include/)
 link_directories(${CMAKE_CURRENT_LIST_DIR}/libs/arm64-v8a)
 
-add_library(llm_mnn SHARED llm_mnn_jni.cpp ../../../../../src/llm.cpp)
+FILE(GLOB SRCS ../../../../../src/*.cpp)
+add_library(llm_mnn SHARED llm_mnn_jni.cpp ${SRCS})
 
 target_link_libraries(llm_mnn MNN MNN_Express log)
diff --git a/android/app/src/main/jni/llm_mnn_jni.cpp b/android/app/src/main/jni/llm_mnn_jni.cpp
@@ -24,12 +24,11 @@ JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved) {
     __android_log_print(ANDROID_LOG_DEBUG, "MNN_DEBUG", "JNI_OnUnload");
 }
 
-JNIEXPORT jboolean JNICALL Java_com_mnn_llm_Chat_Init(JNIEnv* env, jobject thiz, jstring modelDir, jstring tokenizerDir) {
+JNIEXPORT jboolean JNICALL Java_com_mnn_llm_Chat_Init(JNIEnv* env, jobject thiz, jstring modelDir) {
     if (llm->load_progress() < 100) {
         const char* model_dir = env->GetStringUTFChars(modelDir, 0);
-        const char* token_dir = env->GetStringUTFChars(tokenizerDir, 0);
         llm = Llm::createLLM(model_dir);
-        llm->load(model_dir, token_dir);
+        llm->load(model_dir);
     }
     return JNI_TRUE;
 }

diff --git a/demo/cli_demo.cpp b/demo/cli_demo.cpp
@@ -48,12 +48,11 @@ int main(int argc, const char* argv[]) {
     app.add_option("-c,--cpusize", cpusize,"cpu memory size(G), default is 8G.");
     app.add_option("-g,--gpusize", gpusize,"gpu memory size(G)");
 	app.add_option("-m,--model_dir", model_dir, "model directory");
-    app.add_option("-t,--tokenizer_dir", tokenizer_dir, "tokenizer directory");
 
 	CLI11_PARSE(app, argc, argv);
     std::cout << "model path is " << model_dir << std::endl;
     std::unique_ptr<Llm> llm(Llm::createLLM(model_dir));
-    llm->load(model_dir, tokenizer_dir);
+    llm->load(model_dir);
     llm->response("你好");
     return 0;
 }
diff --git a/demo/web_demo.cpp b/demo/web_demo.cpp
@@ -50,13 +50,12 @@ int main(int argc, const char* argv[]) {
     app.add_option("-c,--cpusize", cpusize,"cpu memory size(G), default is 8G.");
     app.add_option("-g,--gpusize", gpusize,"gpu memory size(G)");
 	app.add_option("-m,--model_dir", model_dir, "model directory");
-    app.add_option("-t,--tokenizer_dir", tokenizer_dir, "tokenizer directory");
 
     CLI11_PARSE(app, argc, argv);
 
     std::cout << "model path is " << model_dir << std::endl;
     std::unique_ptr<Llm> llm(Llm::createLLM(model_dir));
-    llm->load(model_dir, tokenizer_dir);
+    llm->load(model_dir);
 
     std::stringstream ss;
     httplib::Server svr;