Skip to content

Commit

Permalink
Update log to only print when input and output characters are different
Browse files Browse the repository at this point in the history
  • Loading branch information
mokeddembillel committed Dec 18, 2024
1 parent d8d2f37 commit 92e41ec
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion convert_hf_to_gguf.py
Original file line number Diff line number Diff line change
Expand Up @@ -529,8 +529,10 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]:
# used for `\n` / `\t` have been manually added in the added tokens
# To avoid unexpected issues - we make sure to encode single-char tokens
if len(token) == 1:
logger.info("Ecode-Decode special characters using AutoTokenizer")
previous_token = token
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
if previous_token != token:
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")

if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token):
toktypes.append(gguf.TokenType.CONTROL)
Expand Down

0 comments on commit 92e41ec

Please sign in to comment.