diff --git a/tests/EncodingTest.php b/tests/EncodingTest.php index 0817c86..b56b962 100644 --- a/tests/EncodingTest.php +++ b/tests/EncodingTest.php @@ -147,5 +147,13 @@ public function testDecodeWithSpecial() $this->assertSame('Hello World<|endoftext|>Hello danny.', $text); } - // TODO: test: encodeOrdinary === encode($text, disallowedSpecial: []) + public function testEncodeOrdinary() + { + $enc = EncodingFactory::createByEncodingName('cl100k_base'); + $tokens1 = $enc->encodeOrdinary('🫡🍣顏文字'); + $tokens2 = $enc->encode('🫡🍣顏文字', disallowedSpecial: []); + + $this->assertSame([9468, 104, 94, 9468, 235, 96, 14167, 237, 88435], $tokens1); + $this->assertSame($tokens1, $tokens2); + } }