added qwen tokenizer test

data/template/tests.py

@@ -10,6 +10,7 @@
     CustomTokenizer,
     CharTokenizer,
     CustomCharTokenizerWithByteFallback,
+    Qwen2Tokenizer
 )
 from argparse import Namespace
 from rich.console import Console
@@ -208,6 +209,22 @@ def test_custom_char_tokenizer_with_byte_fallback(self):
         if os.path.exists(args.custom_chars_file):
             os.remove(args.custom_chars_file)
 
+    def test_qwen2_tokenizer(self):
+        args = Namespace(qwen2_model="qwen2_0p5b")
+        tokenizer = Qwen2Tokenizer(args)
+
+        # Tokenize
+        ids = tokenizer.tokenize(self.sample_text)
+        detokenized = tokenizer.detokenize(ids)
+
+        console.print("[input]Input:[/input]")
+        console.print(self.sample_text, style="input")
+        console.print("[output]Detokenized Output:[/output]")
+        console.print(detokenized, style="output")
+
+        # Assert that detokenized text matches original
+        self.assertEqual(self.sample_text, detokenized)
+
 if __name__ == '__main__':
     run_tests()