allow special characters in decoding

mmoffatt2 · Dec 11, 2024 · 04312f0 · 04312f0
1 parent 9de7104
commit 04312f0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/data/template/tokenizer_options.py b/data/template/tokenizer_options.py
@@ -354,4 +354,4 @@ def detokenize(self, ids):
         Returns:
             str: Decoded string.
         """
-        return self.tokenizer.decode(ids, skip_special_tokens=True)
+        return self.tokenizer.decode(ids)