add testEncodeOrdinary

danny50610 · Aug 20, 2023 · 2de45e4 · 2de45e4
1 parent de8465d
commit 2de45e4
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/tests/EncodingTest.php b/tests/EncodingTest.php
@@ -147,5 +147,13 @@ public function testDecodeWithSpecial()
         $this->assertSame('Hello World<|endoftext|>Hello danny.', $text);
     }
 
-    // TODO: test: encodeOrdinary === encode($text, disallowedSpecial: [])
+    public function testEncodeOrdinary()
+    {
+        $enc = EncodingFactory::createByEncodingName('cl100k_base');
+        $tokens1 = $enc->encodeOrdinary('🫡🍣顏文字');
+        $tokens2 = $enc->encode('🫡🍣顏文字', disallowedSpecial: []);
+
+        $this->assertSame([9468, 104, 94, 9468, 235, 96, 14167, 237, 88435], $tokens1);
+        $this->assertSame($tokens1, $tokens2);
+    }
 }