diff --git a/CMakeLists.txt b/CMakeLists.txt index deee1388..632996232 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,10 @@ -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(mnn-llm) option(BUILD_FOR_ANDROID "Build for android whith mini memory mode." OFF) option(USING_VISUAL_MODEL "Using visual model will need dpes: MNNOpenCV and httplib." OFF) option(DUMP_PROFILE_INFO "Dump profile info when chat." OFF) +option(BUILD_JNI "Build JNI for android app." OFF) if (USING_VISUAL_MODEL) add_definitions(-DUSING_VISUAL_MODEL) @@ -38,6 +39,11 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}/include/ # source files FILE(GLOB SRCS ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) +# jni file +if (BUILD_JNI) + list(APPEND SRCS "${CMAKE_CURRENT_LIST_DIR}/android/app/src/main/jni/llm_mnn_jni.cpp") +endif() + if (MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17") add_compile_options("$<$:/source-charset:utf-8>") @@ -57,6 +63,8 @@ else() endif() endif() +if (NOT (BUILD_JNI)) + add_executable(cli_demo ${CMAKE_CURRENT_LIST_DIR}/demo/cli_demo.cpp) add_executable(tokenizer_demo ${CMAKE_CURRENT_LIST_DIR}/demo/tokenizer_demo.cpp) add_executable(embedding_demo ${CMAKE_CURRENT_LIST_DIR}/demo/embedding_demo.cpp) @@ -110,3 +118,5 @@ else() target_link_libraries(web_demo llm pthread) endif() endif() + +endif() \ No newline at end of file diff --git a/android/app/build.gradle b/android/app/build.gradle index 00705774..80d09ddf 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -10,8 +10,11 @@ android { versionCode 1 versionName "1.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" - ndk { - abiFilters 'arm64-v8a' + externalNativeBuild { + cmake { + arguments "-DBUILD_JNI=TRUE", "-DANDROID_STL=c++_shared" + abiFilters 'arm64-v8a' + } } } buildTypes { @@ -22,8 +25,8 @@ android { } externalNativeBuild { cmake { - version "3.10.2" - path file('src/main/jni/CMakeLists.txt') + version "3.28.1" + path file('../../CMakeLists.txt') } } sourceSets { diff --git a/android/app/src/main/java/com/mnn/llm/Chat.java b/android/app/src/main/java/com/mnn/llm/Chat.java index 24cc21b9..530860e8 100644 --- a/android/app/src/main/java/com/mnn/llm/Chat.java +++ b/android/app/src/main/java/com/mnn/llm/Chat.java @@ -12,6 +12,6 @@ public class Chat implements Serializable { public native void Reset(); static { - System.loadLibrary("llm_mnn"); + System.loadLibrary("llm"); } } diff --git a/android/app/src/main/java/com/mnn/llm/MainActivity.java b/android/app/src/main/java/com/mnn/llm/MainActivity.java index 2831a0b6..d1f6276e 100644 --- a/android/app/src/main/java/com/mnn/llm/MainActivity.java +++ b/android/app/src/main/java/com/mnn/llm/MainActivity.java @@ -60,7 +60,7 @@ public void onItemSelected(AdapterView parent, View view, int position, long mModelName = (String) parent.getItemAtPosition(position); mModelInfo.setText("选择模型:" + mModelName); mModelInfo.setVisibility(View.VISIBLE); - mModelDir = mSearchPath + mModelName; + mModelDir = mSearchPath + mModelName + "/config.json"; } } @Override @@ -120,13 +120,6 @@ public boolean checkModelsReady() { if (!dir.exists()) { return false; } - String[] modelArray = this.getResources().getStringArray(R.array.model_list); - for (int i = 0; i < modelArray.length; i++) { - File model = new File(mModelDir, modelArray[i]); - if (!model.exists()) { - return false; - } - } return true; } diff --git a/android/app/src/main/jni/CMakeLists.txt b/android/app/src/main/jni/CMakeLists.txt deleted file mode 100644 index 246aa329..00000000 --- a/android/app/src/main/jni/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -project(chatglm_mnn) - -cmake_minimum_required(VERSION 3.10) - -# set(MNN_DIR ${CMAKE_SOURCE_DIR}/mnn/${ANDROID_ABI}/lib/cmake/ncnn) -# find_package(MNN REQUIRED) -include_directories(${CMAKE_CURRENT_LIST_DIR}/../../../../../include/) -link_directories(${CMAKE_CURRENT_LIST_DIR}/libs/arm64-v8a) - -add_definitions(-DUSING_DISK_EMBED) -FILE(GLOB SRCS ../../../../../src/*.cpp) -add_library(llm_mnn SHARED llm_mnn_jni.cpp ${SRCS}) - -target_link_libraries(llm_mnn MNN MNN_Express log) \ No newline at end of file diff --git a/android/app/src/main/jni/libs/arm64-v8a/.gitkeep b/android/app/src/main/jni/libs/arm64-v8a/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/android/app/src/main/jni/llm_mnn_jni.cpp b/android/app/src/main/jni/llm_mnn_jni.cpp index a1481b82..957dc15b 100644 --- a/android/app/src/main/jni/llm_mnn_jni.cpp +++ b/android/app/src/main/jni/llm_mnn_jni.cpp @@ -28,13 +28,13 @@ JNIEXPORT jboolean JNICALL Java_com_mnn_llm_Chat_Init(JNIEnv* env, jobject thiz, const char* model_dir = env->GetStringUTFChars(modelDir, 0); if (!llm.get()) { llm.reset(Llm::createLLM(model_dir)); - llm->load(model_dir); + llm->load(); } return JNI_TRUE; } JNIEXPORT jboolean JNICALL Java_com_mnn_llm_Chat_Ready(JNIEnv* env, jobject thiz) { - if (llm.get() && llm->load_progress() >= 100) { + if (llm.get()) { return JNI_TRUE; } return JNI_FALSE; @@ -42,7 +42,7 @@ JNIEXPORT jboolean JNICALL Java_com_mnn_llm_Chat_Ready(JNIEnv* env, jobject thiz JNIEXPORT jfloat JNICALL Java_com_mnn_llm_Chat_Progress(JNIEnv* env, jobject thiz) { if (!llm.get()) return jfloat(0); - return jfloat(llm->load_progress()); + return jfloat(llm->load_progress_); } JNIEXPORT jstring JNICALL Java_com_mnn_llm_Chat_Submit(JNIEnv* env, jobject thiz, jstring inputStr) { @@ -59,13 +59,6 @@ JNIEXPORT jstring JNICALL Java_com_mnn_llm_Chat_Submit(JNIEnv* env, jobject thiz return result; } -/* -JNIEXPORT jstring JNICALL Java_com_mnn_chatglm_Chat_Response(JNIEnv* env, jobject thiz) { - jstring result = env->NewStringUTF(response_buffer.str().c_str()); - return result; -} -*/ - JNIEXPORT jbyteArray JNICALL Java_com_mnn_llm_Chat_Response(JNIEnv* env, jobject thiz) { auto len = response_buffer.str().size(); jbyteArray res = env->NewByteArray(len); @@ -78,7 +71,7 @@ JNIEXPORT void JNICALL Java_com_mnn_llm_Chat_Done(JNIEnv* env, jobject thiz) { } JNIEXPORT void JNICALL Java_com_mnn_llm_Chat_Reset(JNIEnv* env, jobject thiz) { - llm->reset(); + // llm->reset(); } } // extern "C" \ No newline at end of file diff --git a/android/app/src/main/res/values/models.xml b/android/app/src/main/res/values/models.xml deleted file mode 100644 index d5e7d4c5..00000000 --- a/android/app/src/main/res/values/models.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - block_0.mnn - block_1.mnn - block_2.mnn - block_3.mnn - block_4.mnn - block_5.mnn - block_6.mnn - block_7.mnn - block_8.mnn - block_9.mnn - block_10.mnn - block_11.mnn - block_12.mnn - block_13.mnn - block_14.mnn - block_15.mnn - block_16.mnn - block_17.mnn - block_18.mnn - block_19.mnn - block_20.mnn - block_21.mnn - block_22.mnn - block_23.mnn - embeddings_bf16.bin - lm.mnn - tokenizer.txt - - \ No newline at end of file diff --git a/android/build.gradle b/android/build.gradle index 95a39961..1a4faacf 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:4.0.1' + classpath 'com.android.tools.build:gradle:7.0.1' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/android/gradle/wrapper/gradle-wrapper.properties b/android/gradle/wrapper/gradle-wrapper.properties index da9702f9..2e6e5897 100644 --- a/android/gradle/wrapper/gradle-wrapper.properties +++ b/android/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/include/llm.hpp b/include/llm.hpp index 505e3216..a57ae49e 100644 --- a/include/llm.hpp +++ b/include/llm.hpp @@ -247,6 +247,7 @@ class Llm { // time int64_t prefill_us_ = 0; int64_t decode_us_ = 0; + float load_progress_ = 0.f; bool is_single_ = true; bool is_disk_embedding_ = true; std::shared_ptr config_; @@ -276,7 +277,7 @@ class Lvlm : public Llm { img_pad_ = config->llm_config_.value("img_pad", img_pad_); } ~Lvlm() { visual_module_.reset(); } - virtual void load(); + virtual void load() override; private: int img_size_ = 448, imgpad_len_ = 256, img_start_ = 151857, img_end_ = 151858, img_pad_ = 151859; std::shared_ptr visual_module_; diff --git a/src/llm.cpp b/src/llm.cpp index 882613ac..8e63f675 100644 --- a/src/llm.cpp +++ b/src/llm.cpp @@ -109,6 +109,7 @@ void Llm::load() { MNN_PRINT("Done!\n"); } } + load_progress_ = 100.f; } VARP Llm::forward(const std::vector& input_ids) {