fix qwen2 tokenizer name

InternLM · Oct 16, 2024 · 6881a97 · 6881a97
1 parent 4a1b201
commit 6881a97
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/xtuner/dataset/utils.py b/xtuner/dataset/utils.py
@@ -14,7 +14,7 @@
 
 def get_bos_eos_token_ids(tokenizer):
     if tokenizer.__class__.__name__ in [
-            'QWenTokenizer', 'QWen2Tokenizer', 'Qwen2TokenizerFast'
+            'QWenTokenizer', 'Qwen2Tokenizer', 'Qwen2TokenizerFast'
     ]:
         bos_token_id = []
         eos_token_id = tokenizer.eos_token_id