Track line number for each token

Author: theodorejb

composer.json

@@ -28,7 +28,7 @@
         }
     },
     "scripts": {
-        "analyze": "phpstan analyze src --level 10",
+        "analyze": "phpstan analyze --level 10 src test",
         "test": "vendor/bin/phpunit test"
     },
     "config": {

src/Phlexer/Phlexer.php

@@ -49,6 +49,7 @@ public function getNextToken(): ?Token
             return null;
         }
 
+        $line = substr_count($this->text, "\n", 0, $this->cursor + 1) + 1;
         $subject = substr($this->text, $this->cursor);
 
         foreach ($this->rules as $rule) {
@@ -66,7 +67,7 @@ public function getNextToken(): ?Token
                     return $this->getNextToken();
                 }
 
-                return new Token($tokenName, $this->yytext);
+                return new Token($tokenName, $this->yytext, $line);
             }
         }
 

src/Phlexer/Token.php

@@ -7,5 +7,6 @@
     public function __construct(
         public string $name,
         public string $text,
+        public int $line,
     ) {}
 }

test/LexerTest.php

@@ -7,15 +7,32 @@
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\TestCase;
 
+/**
+ * @phpstan-type SpecToken array{name: string, text: string}
+ * @phpstan-type SpecArr array{it: string, template: string, expected: list<SpecToken>}
+ */
 class LexerTest extends TestCase
 {
+    /**
+     * @return list<array{0: SpecArr}>
+     */
     public static function jsonSpecProvider(): array
     {
         $filename = 'vendor/jbboehr/handlebars-spec/spec/tokenizer.json';
-        $json = json_decode(file_get_contents($filename), true);
+        $contents = file_get_contents($filename);
+
+        if ($contents === false) {
+            throw new \Exception("Failed to read file {$filename}");
+        }
+
+        /** @var list<SpecArr> $json */
+        $json = json_decode($contents, true);
         return array_map(fn(array $d): array => [$d], $json);
     }
 
+    /**
+     * @param SpecArr $spec
+     */
     #[DataProvider("jsonSpecProvider")]
     public function testSpecs(array $spec): void
     {
@@ -32,4 +49,25 @@ public function testSpecs(array $spec): void
         $actual = array_map($toJson, $lexer->tokenize($spec['template']));
         $this->assertSame($spec['expected'], $actual);
     }
+
+    public function testLineNumbers(): void
+    {
+        $template = <<<_tpl
+            This
+            is a {{template}}
+            with multiple
+            {{lines}}
+            _tpl;
+        $expected = [
+            new Token('CONTENT', "This\nis a ", 1),
+            new Token('OPEN', '{{', 2),
+            new Token('ID', 'template', 2),
+            new Token('CLOSE', '}}', 2),
+            new Token('CONTENT', "\nwith multiple\n", 3),
+            new Token('OPEN', '{{', 4),
+            new Token('ID', 'lines', 4),
+            new Token('CLOSE', '}}', 4),
+        ];
+        $this->assertEquals($expected, (new Lexer())->tokenize($template));
+    }
 }