Reduce all info re-query gap to 500 ms.

Cargo.toml

@@ -24,6 +24,9 @@ itertools = "0.13"
 memmap2 = "0.9"
 safetensors = "0.4"
 
+[workspace.dependencies.ai00-core]
+path = "crates/ai00-core"
+
 [workspace.dependencies.web-rwkv]
 # path = "../web-rwkv"
 default-features = false

crates/ai00-server/Cargo.toml

@@ -26,6 +26,9 @@ tokio = { version = "1", features = ["full"] }
 toml = "0.8.6"
 zip-extract = "0.1"
 
+[dependencies.ai00-core]
+workspace = true
+
 [dependencies.anyhow]
 workspace = true
 
@@ -61,6 +64,3 @@ features = [
     "sse",
 ]
 version = "0.67"
-
-[dependencies.ai00-core]
-path = "../ai00-core"

crates/ai00-server/src/api/mod.rs

@@ -32,12 +32,12 @@ pub async fn request_info(sender: Sender<ThreadRequest>, sleep: Duration) -> Run
 
 pub async fn request_info_stream(
     sender: Sender<ThreadRequest>,
-    info_sender: Sender<RuntimeInfo>,
+    stream: Sender<RuntimeInfo>,
     sleep: Duration,
 ) {
     loop {
         if let Ok(_info) = try_request_info(sender.clone()).await {
-            if info_sender.send(_info).is_err() {
+            if stream.send(_info).is_err() {
                 break;
             }
         }

crates/ai00-server/src/api/oai/chat.rs

@@ -184,7 +184,7 @@ struct PartialChatResponse {
 
 async fn respond_one(depot: &mut Depot, request: ChatRequest, res: &mut Response) {
     let ThreadState { sender, .. } = depot.obtain::<ThreadState>().unwrap();
-    let info = request_info(sender.clone(), Duration::from_secs(1)).await;
+    let info = request_info(sender.clone(), Duration::from_millis(500)).await;
     let model_name = info.reload.model_path.to_string_lossy().into_owned();
 
     let (token_sender, token_receiver) = flume::unbounded();
@@ -233,7 +233,7 @@ async fn respond_one(depot: &mut Depot, request: ChatRequest, res: &mut Response
 
 async fn respond_stream(depot: &mut Depot, request: ChatRequest, res: &mut Response) {
     let ThreadState { sender, .. } = depot.obtain::<ThreadState>().unwrap();
-    let info = request_info(sender.clone(), Duration::from_secs(1)).await;
+    let info = request_info(sender.clone(), Duration::from_millis(500)).await;
     let model_name = info.reload.model_path.to_string_lossy().into_owned();
 
     let (token_sender, token_receiver) = flume::unbounded();

crates/ai00-server/src/api/oai/completion.rs

@@ -131,7 +131,7 @@ pub struct PartialCompletionResponse {
 
 async fn respond_one(depot: &mut Depot, request: CompletionRequest, res: &mut Response) {
     let ThreadState { sender, .. } = depot.obtain::<ThreadState>().unwrap();
-    let info = request_info(sender.clone(), Duration::from_secs(1)).await;
+    let info = request_info(sender.clone(), Duration::from_millis(500)).await;
     let model_name = info.reload.model_path.to_string_lossy().into_owned();
 
     let (token_sender, token_receiver) = flume::unbounded();
@@ -177,7 +177,7 @@ async fn respond_one(depot: &mut Depot, request: CompletionRequest, res: &mut Re
 
 async fn respond_stream(depot: &mut Depot, request: CompletionRequest, res: &mut Response) {
     let ThreadState { sender, .. } = depot.obtain::<ThreadState>().unwrap();
-    let info = request_info(sender.clone(), Duration::from_secs(1)).await;
+    let info = request_info(sender.clone(), Duration::from_millis(500)).await;
     let model_name = info.reload.model_path.to_string_lossy().into_owned();
 
     let (token_sender, token_receiver) = flume::unbounded();

crates/ai00-server/src/api/oai/embedding.rs

@@ -57,7 +57,7 @@ pub async fn embeddings(
 ) -> Json<EmbeddingResponse> {
     let request = req.to_owned(); // req.parse_json::<EmbeddingRequest>().await.unwrap();
     let ThreadState { sender, .. } = depot.obtain::<ThreadState>().unwrap();
-    let info = request_info(sender.clone(), Duration::from_secs(1)).await;
+    let info = request_info(sender.clone(), Duration::from_millis(500)).await;
     let model_name = info.reload.model_path.to_string_lossy().into_owned();
 
     let (token_sender, token_receiver) = flume::unbounded();

crates/ai00-server/src/api/oai/info.rs

@@ -23,7 +23,7 @@ pub struct ModelResponse {
 #[endpoint]
 pub async fn models(depot: &mut Depot) -> Json<ModelResponse> {
     let ThreadState { sender, .. } = depot.obtain::<ThreadState>().unwrap();
-    let info = request_info(sender.to_owned(), Duration::from_secs(1)).await;
+    let info = request_info(sender.to_owned(), Duration::from_millis(500)).await;
     let model_name = info
         .reload
         .model_path