Merge pull request #432 from Sanctum-AI/fix-rewind

Fix rewind crash
rustformers · Oct 31, 2023 · fb48749 · fb48749
2 parents db31c2d + cd97c9d
commit fb48749
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/crates/llm-base/src/inference_session.rs b/crates/llm-base/src/inference_session.rs
@@ -360,7 +360,7 @@ impl InferenceSession {
         }
 
         // Remove the tokens from self.tokens.
-        let token_start = self.n_past - num;
+        let token_start = self.tokens.len() - num;
         let deleted_tokens: Vec<_> = self.tokens.drain(token_start..).collect();
 
         // Remove the corresponding chars from decoded