From 2de45e47a461f1d870accf5f6131a0d11dcb38c9 Mon Sep 17 00:00:00 2001
From: "Chan, Danny" <danny50610@yahoo.com.tw>
Date: Sun, 20 Aug 2023 08:46:38 +0000
Subject: [PATCH] add testEncodeOrdinary

---
 tests/EncodingTest.php | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/EncodingTest.php b/tests/EncodingTest.php
index 0817c86..b56b962 100644
--- a/tests/EncodingTest.php
+++ b/tests/EncodingTest.php
@@ -147,5 +147,13 @@ public function testDecodeWithSpecial()
         $this->assertSame('Hello World<|endoftext|>Hello danny.', $text);
     }
 
-    // TODO: test: encodeOrdinary === encode($text, disallowedSpecial: [])
+    public function testEncodeOrdinary()
+    {
+        $enc = EncodingFactory::createByEncodingName('cl100k_base');
+        $tokens1 = $enc->encodeOrdinary('🫡🍣顏文字');
+        $tokens2 = $enc->encode('🫡🍣顏文字', disallowedSpecial: []);
+
+        $this->assertSame([9468, 104, 94, 9468, 235, 96, 14167, 237, 88435], $tokens1);
+        $this->assertSame($tokens1, $tokens2);
+    }
 }