diff --git a/paddlemix/auto/tokenizer.py b/paddlemix/auto/tokenizer.py index 6d90153b6..721f85380 100644 --- a/paddlemix/auto/tokenizer.py +++ b/paddlemix/auto/tokenizer.py @@ -15,7 +15,7 @@ import io import json import os - +from paddlemix.models.qwen_vl.tokenizer import QWenVLTokenizer import yaml from paddlenlp.transformers import AutoTokenizer #fix paddlenlp 3.0b3 auto @@ -24,6 +24,8 @@ try: AutoTokenizer.register(LlavaConfig, LLavaTokenizer) print('LLavaTokenizer register success!!!!') + AutoTokenizer.register("qwen_vl", QWenVLTokenizer) + print('QWenVLTokenizer register success!!!!') except: pass