[update]

llm-llm add async list.
llm-yolo add async list.
add llm-yolo model config  output_type.

Author: dianjixz

projects/llm_framework/main_llm/src/main.cpp

@@ -16,6 +16,7 @@
 #include <stdexcept>
 #include <semaphore.h>
 #include "../../../../SDK/components/utilities/include/sample_log.h"
+#include "thread_safe_list.h"
 using namespace StackFlows;
 #ifdef ENABLE_BACKWARD
 #define BACKWARD_HAS_DW 1
@@ -58,11 +59,9 @@ class llm_task {
     task_callback_t out_callback_;
     bool enoutput_;
     bool enstream_;
-    sem_t inference_semaphore;
-    std::atomic_int inference_status_;
+
     std::unique_ptr<std::thread> inference_run_;
-    std::atomic_bool is_running_;
-    std::string _inference_msg;
+    thread_safe::list<std::string> async_list_;
 
     void set_output(task_callback_t out_callback)
     {
@@ -223,24 +222,26 @@ class llm_task {
 
     void run()
     {
-        sem_wait(&inference_semaphore);
-        while (is_running_) {
+        std::string par;
+        for (;;) {
             {
-                inference(_inference_msg);
-                inference_status_--;
-                sem_wait(&inference_semaphore);
+                par = async_list_.get();
+                if (par.empty()) break;
+                inference(par);
             }
         }
     }
 
     int inference_async(const std::string &msg)
     {
-        if (inference_status_ == INFERENCE_NONE) {
-            _inference_msg    = msg;
-            inference_status_ = INFERENCE_RUNNING;
-            sem_post(&inference_semaphore);
+        if (msg.empty()) return -1;
+        if (async_list_.size() < 3) {
+            std::string par = msg;
+            async_list_.put(par);
+        } else {
+            SLOGE("inference list is full\n");
         }
-        return inference_status_;
+        return async_list_.size();
     }
 
     void inference(const std::string &msg)
@@ -286,7 +287,8 @@ class llm_task {
 
     bool pause()
     {
-        lLaMa_->Stop();
+        if(lLaMa_)
+            lLaMa_->Stop();
         return true;
     }
 
@@ -314,25 +316,38 @@ class llm_task {
 
     llm_task(const std::string &workid) : tokenizer_server_flage_(false), port_(getNextPort())
     {
-        inference_status_ = INFERENCE_NONE;
-        sem_init(&inference_semaphore, 0, 0);
-        is_running_    = true;
         inference_run_ = std::make_unique<std::thread>(std::bind(&llm_task::run, this));
     }
 
+    void start()
+    {
+        if (!inference_run_) {
+            inference_run_ = std::make_unique<std::thread>(std::bind(&llm_task::run, this));
+        }
+    }
+
+    void stop()
+    {
+        if (inference_run_) {
+            std::string par;
+            async_list_.put(par);
+            if(lLaMa_)
+                lLaMa_->Stop();
+            inference_run_->join();
+            inference_run_.reset();
+        }
+    }    
+
     ~llm_task()
     {
-        is_running_ = false;
-        sem_post(&inference_semaphore);
-        if (inference_run_) inference_run_->join();
+        stop();
         if (tokenizer_pid_ != -1) {
             kill(tokenizer_pid_, SIGTERM);
             waitpid(tokenizer_pid_, nullptr, WNOHANG);
         }
         if (lLaMa_) {
             lLaMa_->Deinit();
         }
-        sem_destroy(&inference_semaphore);
     }
 };
 
@@ -647,10 +662,9 @@ class llm_llm : public StackFlow {
             send("None", "None", error_body, work_id);
             return -1;
         }
-        task_pause(llm_task_[work_id_num], get_channel(work_id_num));
+        llm_task_[work_id_num]->stop();
         auto llm_channel = get_channel(work_id_num);
         llm_channel->stop_subscriber("");
-        llm_task_[work_id_num]->lLaMa_->Stop();
         llm_task_.erase(work_id_num);
         send("None", "None", LLM_NO_ERROR, work_id);
         return 0;
@@ -663,6 +677,7 @@ class llm_llm : public StackFlow {
             if (iteam == llm_task_.end()) {
                 break;
             }
+            iteam->second->stop();
             get_channel(iteam->first)->stop_subscriber("");
             iteam->second.reset();
             llm_task_.erase(iteam->first);

projects/llm_framework/main_yolo/SConstruct

@@ -16,7 +16,7 @@ LDFLAGS = []
 LINK_SEARCH_PATH = []
 STATIC_FILES = []
 
-DEFINITIONS += ['-std=c++17']
+DEFINITIONS += ['-std=c++17', '-O2']
 LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
 LINK_SEARCH_PATH += [ADir('../static_lib')]
 REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
@@ -30,6 +30,15 @@ STATIC_LIB += static_file * 2
 STATIC_FILES += Glob('mode_*.json')
 
 
+
+# REQUIREMENTS += ['Backward_cpp']
+# DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'),
+#                 AFile('../static_lib/libelf.so.1'),
+#                 AFile('../static_lib/libz.so.1'),
+#                 AFile('../static_lib/liblzma.so.5'),
+#                 AFile('../static_lib/libbz2.so.1.0')]
+# DEFINITIONS += ["-DENABLE_BACKWARD"]
+
 env['COMPONENTS'].append({'target':'llm_yolo',
                           'SRCS':SRCS,
                           'INCLUDE':INCLUDE,

projects/llm_framework/main_yolo/mode_yolo11n-hand-pose.json

@@ -8,7 +8,8 @@
         "yolo.jpeg.base64"
     ],
     "output_type":[
-        "yolo.yolobox"
+        "yolo.box",
+        "yolo.boxV2"
     ],
     "mode_param":{
         "yolo_model":"yolo11n-hand-pose.axmodel",

projects/llm_framework/main_yolo/mode_yolo11n-pose.json

@@ -8,7 +8,8 @@
         "yolo.jpeg.base64"
     ],
     "output_type":[
-        "yolo.yolobox"
+        "yolo.box",
+        "yolo.boxV2"
     ],
     "mode_param":{
         "yolo_model":"yolo11n-pose.axmodel",

projects/llm_framework/main_yolo/mode_yolo11n-seg.json

@@ -8,7 +8,8 @@
         "yolo.jpeg.base64"
     ],
     "output_type":[
-        "yolo.yolobox"
+        "yolo.box",
+        "yolo.boxV2"
     ],
     "mode_param":{
         "yolo_model":"yolo11n-seg.axmodel",

projects/llm_framework/main_yolo/mode_yolo11n.json

@@ -8,7 +8,8 @@
         "yolo.jpeg.base64"
     ],
     "output_type":[
-        "yolo.yolobox"
+        "yolo.box",
+        "yolo.boxV2"
     ],
     "mode_param":{
         "yolo_model":"yolo11n.axmodel",