fix_tokenizer_snapshot_download_bug (vllm-project#4493)

alexm-redhat · Apr 30, 2024 · 6ad58f4 · 6ad58f4
1 parent dd1a50a
commit 6ad58f4
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py
@@ -79,7 +79,7 @@ def get_tokenizer(
                 revision=revision,
                 local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
                 # Ignore weights - we only need the tokenizer.
-                ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
+                ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
             tokenizer_name = tokenizer_path
 
     if tokenizer_mode == "slow":