Skip to content

Commit 4f80e76

Browse files
theodorejbtheodorejb
committed
Track column position for each token
1 parent 9172bea commit 4f80e76

File tree

8 files changed

+159
-28
lines changed

8 files changed

+159
-28
lines changed

‎composer.lock‎

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/Ast/SourceLocation.php‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
readonly class SourceLocation
66
{
77
public function __construct(
8-
public string $source,
98
public Position $start,
109
public Position $end,
1110
) {}

‎src/ParserAbstract.php‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use DevTheorem\HandlebarsParser\Ast\InverseChain;
1717
use DevTheorem\HandlebarsParser\Ast\Literal;
1818
use DevTheorem\HandlebarsParser\Ast\MustacheStatement;
19+
use DevTheorem\HandlebarsParser\Ast\Node;
1920
use DevTheorem\HandlebarsParser\Ast\OpenBlock;
2021
use DevTheorem\HandlebarsParser\Ast\OpenHelper;
2122
use DevTheorem\HandlebarsParser\Ast\OpenPartialBlock;
@@ -167,13 +168,14 @@ protected function postprocessTokens(array $tokens): array
167168

168169
if ($numTokens === 0) {
169170
// empty input - just add sentinel token
170-
return [new Token(Lexer::T_EOF, "\0", 0)];
171+
return [new Token(Lexer::T_EOF, "\0", 0, 0)];
171172
}
172173

173174
$lastToken = $tokens[$numTokens - 1];
174175

175176
// Add sentinel token
176-
$tokens[] = new Token(Lexer::T_EOF, "\0", $lastToken->line);
177+
$column = $lastToken->column + strlen($lastToken->text);
178+
$tokens[] = new Token(Lexer::T_EOF, "\0", $lastToken->line, $column);
177179
return $tokens;
178180
}
179181

@@ -384,6 +386,12 @@ protected function getErrorMessage(int $symbol, int $state, int $line): string
384386
return "Parse error on line $line" . $expectedString . ', got ' . $this->symbolToName[$symbol];
385387
}
386388

389+
private function getNodeError(string $message, Node $node): string
390+
{
391+
$start = $node->loc->start;
392+
return $message . ' - ' . $start->line . ':' . $start->column;
393+
}
394+
387395
/**
388396
* Get limited number of expected tokens in given state.
389397
*
@@ -498,14 +506,11 @@ protected function locInfo(int $tokenStartPos, int $tokenEndPos): SourceLocation
498506
{
499507
$startToken = $this->tokens[$tokenStartPos];
500508
$endToken = $this->tokens[$tokenEndPos];
501-
$source = '';
502509

503-
for ($i = $tokenStartPos; $i <= $tokenEndPos; $i++) {
504-
$token = $this->tokens[$i];
505-
$source .= $token->text;
506-
}
510+
$start = new Position($startToken->line, $startToken->column);
511+
$end = new Position($endToken->line, $endToken->column + strlen($endToken->text));
507512

508-
return new SourceLocation($source, new Position($startToken->line, -1), new Position($endToken->line, -1));
513+
return new SourceLocation($start, $end);
509514
}
510515

511516
protected function id(string $token): string
@@ -537,9 +542,9 @@ protected function prepareProgram(array $statements, ?SourceLocation $loc = null
537542
if ($statements) {
538543
$firstLoc = $statements[0]->loc;
539544
$lastLoc = $statements[count($statements) - 1]->loc;
540-
$loc = new SourceLocation($firstLoc->source, $firstLoc->start, $lastLoc->end);
545+
$loc = new SourceLocation($firstLoc->start, $lastLoc->end);
541546
} else {
542-
$loc = new SourceLocation('', new Position(0, -1), new Position(0, -1));
547+
$loc = new SourceLocation(new Position(0, 0), new Position(0, 0));
543548
}
544549
}
545550

@@ -579,7 +584,8 @@ private function validateClose(OpenHelper $open, CloseBlock|string $close): void
579584
}
580585

581586
if ($open->path->original !== $close) {
582-
throw new \Exception("{$open->path->original} doesn't match {$close}");
587+
$msg = $this->getNodeError("{$open->path->original} doesn't match {$close}", $open->path);
588+
throw new \Exception($msg);
583589
}
584590
}
585591

@@ -610,7 +616,8 @@ protected function preparePath(bool $data, SubExpression|null $sexpr, array $par
610616

611617
if (!$isLiteral && ($part === '..' || $part === '.' || $part === 'this')) {
612618
if (count($tail) > 0) {
613-
throw new \Exception('Invalid path: ' . $original);
619+
$msg = $this->getNodeError("Invalid path: $original", new Node('', $loc));
620+
throw new \Exception($msg);
614621
} elseif ($part === '..') {
615622
$depth++;
616623
}

‎src/Phlexer/Phlexer.php‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ public function getNextToken(): ?Token
5757
return null;
5858
}
5959

60-
$line = substr_count($this->text, "\n", 0, $this->cursor + 1) + 1;
6160
$subject = substr($this->text, $this->cursor);
6261

6362
foreach ($this->rules as $rule) {
@@ -66,6 +65,7 @@ public function getNextToken(): ?Token
6665
}
6766

6867
if (preg_match("/\\A{$rule->pattern}/", $subject, $matches)) {
68+
[$line, $column] = $this->getPosition();
6969
$this->yytext = $matches[0];
7070
$this->cursor += strlen($this->yytext);
7171
$tokenName = ($rule->handler)();
@@ -75,13 +75,37 @@ public function getNextToken(): ?Token
7575
return $this->getNextToken();
7676
}
7777

78-
return new Token($tokenName, $this->yytext, $line);
78+
return new Token($tokenName, $this->yytext, $line, $column);
7979
}
8080
}
8181

8282
throw new \Exception('Unexpected token: "' . $subject[0] . '"');
8383
}
8484

85+
/**
86+
* @return array{int, int}
87+
*/
88+
private function getPosition(): array
89+
{
90+
$line = 1;
91+
$column = -1;
92+
93+
for ($i = 0; $i < $this->cursor + 1; $i++) {
94+
if ($this->text[$i] === "\n") {
95+
$line++;
96+
$column = -1;
97+
} else {
98+
$column++;
99+
}
100+
}
101+
102+
if ($column === -1) {
103+
$column = 0;
104+
}
105+
106+
return [$line, $column];
107+
}
108+
85109
protected function pushState(string $state): void
86110
{
87111
$this->states[] = $state;

‎src/Phlexer/Token.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@ public function __construct(
88
public string $name,
99
public string $text,
1010
public int $line,
11+
public int $column,
1112
) {}
1213
}

‎tests/LexerTest.php‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,14 @@ public function testLineNumbers(): void
5959
{{lines}}
6060
_tpl;
6161
$expected = [
62-
new Token(Lexer::T_CONTENT, "This\nis a ", 1),
63-
new Token(Lexer::T_OPEN, '{{', 2),
64-
new Token(Lexer::T_ID, 'template', 2),
65-
new Token(Lexer::T_CLOSE, '}}', 2),
66-
new Token(Lexer::T_CONTENT, "\nwith multiple\n", 3),
67-
new Token(Lexer::T_OPEN, '{{', 4),
68-
new Token(Lexer::T_ID, 'lines', 4),
69-
new Token(Lexer::T_CLOSE, '}}', 4),
62+
new Token(Lexer::T_CONTENT, "This\nis a ", 1, 0),
63+
new Token(Lexer::T_OPEN, '{{', 2, 5),
64+
new Token(Lexer::T_ID, 'template', 2, 7),
65+
new Token(Lexer::T_CLOSE, '}}', 2, 15),
66+
new Token(Lexer::T_CONTENT, "\nwith multiple\n", 3, 0),
67+
new Token(Lexer::T_OPEN, '{{', 4, 0),
68+
new Token(Lexer::T_ID, 'lines', 4, 2),
69+
new Token(Lexer::T_CLOSE, '}}', 4, 7),
7070
];
7171
$this->assertEquals($expected, (new Lexer())->tokenize($template));
7272
}

‎tests/ParserTest.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,24 @@
1313
*/
1414
class ParserTest extends TestCase
1515
{
16+
public function testParse(): void
17+
{
18+
/*
19+
* Compare to the following Handlebars.js code:
20+
*
21+
* import {parse} from '@handlebars/parser';
22+
* const template = '{{foo bar}}';
23+
* const ast = parse(template);
24+
* process.stdout.write(JSON.stringify(ast, undefined, 4));
25+
*/
26+
27+
$parser = (new ParserFactory())->create();
28+
$result = $parser->parse('{{foo bar}}');
29+
30+
$actual = json_encode($result, JSON_PRETTY_PRINT);
31+
$this->assertSame(file_get_contents('tests/test1.json'), $actual);
32+
}
33+
1634
/**
1735
* @return \Generator<array{0: SpecArr}>
1836
*/

‎tests/test1.json‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{
2+
"type": "Program",
3+
"loc": {
4+
"start": {
5+
"line": 1,
6+
"column": 0
7+
},
8+
"end": {
9+
"line": 1,
10+
"column": 11
11+
}
12+
},
13+
"body": [
14+
{
15+
"type": "MustacheStatement",
16+
"loc": {
17+
"start": {
18+
"line": 1,
19+
"column": 0
20+
},
21+
"end": {
22+
"line": 1,
23+
"column": 11
24+
}
25+
},
26+
"path": {
27+
"type": "PathExpression",
28+
"loc": {
29+
"start": {
30+
"line": 1,
31+
"column": 2
32+
},
33+
"end": {
34+
"line": 1,
35+
"column": 5
36+
}
37+
},
38+
"this_": false,
39+
"data": false,
40+
"depth": 0,
41+
"head": "foo",
42+
"tail": [],
43+
"parts": [
44+
"foo"
45+
],
46+
"original": "foo"
47+
},
48+
"params": [
49+
{
50+
"type": "PathExpression",
51+
"loc": {
52+
"start": {
53+
"line": 1,
54+
"column": 6
55+
},
56+
"end": {
57+
"line": 1,
58+
"column": 9
59+
}
60+
},
61+
"this_": false,
62+
"data": false,
63+
"depth": 0,
64+
"head": "bar",
65+
"tail": [],
66+
"parts": [
67+
"bar"
68+
],
69+
"original": "bar"
70+
}
71+
],
72+
"hash": null,
73+
"escaped": true,
74+
"strip": {
75+
"open": false,
76+
"close": false
77+
}
78+
}
79+
],
80+
"blockParams": [],
81+
"chained": false
82+
}

0 commit comments

Comments
 (0)