Update _utils.py

unslothai · Jul 3, 2024 · 8526c3e · 8526c3e
1 parent 9b2beba
commit 8526c3e
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
@@ -21,6 +21,11 @@
 warnings.filterwarnings(action = "ignore", category = UserWarning,    module = "transformers")
 warnings.filterwarnings(action = "ignore", category = FutureWarning,  module = "accelerate")
 warnings.filterwarnings(action = "ignore", category = FutureWarning,  module = "huggingface_hub")
+
+# Stop "Special tokens have been added in the vocabulary, ..."
+import logging
+logging.getLogger("transformers.tokenization_utils_base").setLevel(logging.CRITICAL+1)
+
 import bitsandbytes as bnb
 from transformers.models.llama.modeling_llama import logger
 from transformers import AutoTokenizer