packages/bison-parser/tests/Unit/Scanner/ScannerTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Scanner;
6
7use BisonParser\Ast\Location;
8use BisonParser\Scanner\CodeReader;
9use BisonParser\Scanner\Cursor;
10use BisonParser\Scanner\Directives;
11use BisonParser\Scanner\Escapes;
12use BisonParser\Scanner\Scanner;
13use BisonParser\Scanner\Token;
14use BisonParser\Scanner\TokenKind;
15use BisonParser\SyntaxException;
16use PHPUnit\Framework\Attributes\CoversClass;
17use PHPUnit\Framework\Attributes\Small;
18use PHPUnit\Framework\Attributes\UsesClass;
19use PHPUnit\Framework\TestCase;
20
21#[CoversClass(Scanner::class)]
22#[UsesClass(CodeReader::class)]
23#[UsesClass(Cursor::class)]
24#[UsesClass(Directives::class)]
25#[UsesClass(Escapes::class)]
26#[UsesClass(Location::class)]
27#[UsesClass(SyntaxException::class)]
28#[UsesClass(Token::class)]
29#[UsesClass(TokenKind::class)]
30#[Small]
31final class ScannerTest extends TestCase
32{
33 public function testScan(): void
34 {
35 $tokens = (new Scanner())->scan("%token NUM 258 \"number\"\n%%\nexpr[e]: expr '+' NUM { \$\$ = \$1 + \$3; } | NUM ;\n%%\nint main() {}\n");
36
37 self::assertSame(
38 [
39 [TokenKind::Directive, 'token', '1:1'],
40 [TokenKind::Identifier, 'NUM', '1:8'],
41 [TokenKind::Integer, '258', '1:12'],
42 [TokenKind::String, 'number', '1:16'],
43 [TokenKind::Section, '%%', '2:1'],
44 [TokenKind::IdentifierColon, 'expr', '3:1'],
45 [TokenKind::BracketedIdentifier, 'e', '3:5'],
46 [TokenKind::Colon, ':', '3:8'],
47 [TokenKind::Identifier, 'expr', '3:10'],
48 [TokenKind::CharLiteral, '+', '3:15'],
49 [TokenKind::Identifier, 'NUM', '3:19'],
50 [TokenKind::Code, ' $$ = $1 + $3; ', '3:23'],
51 [TokenKind::Pipe, '|', '3:41'],
52 [TokenKind::Identifier, 'NUM', '3:43'],
53 [TokenKind::Semicolon, ';', '3:47'],
54 [TokenKind::Section, '%%', '4:1'],
55 [TokenKind::Epilogue, "\nint main() {}\n", '4:3'],
56 [TokenKind::End, '', '6:1'],
57 ],
58 array_map(static fn (Token $token): array => [$token->kind, $token->text, (string) $token->location], $tokens),
59 );
60 }
61
62 public function testScanEndsWithoutAnEpilogue(): void
63 {
64 $tokens = (new Scanner())->scan('%% a: ;');
65
66 self::assertSame([TokenKind::Section, TokenKind::IdentifierColon, TokenKind::Colon, TokenKind::Semicolon, TokenKind::End], array_map(static fn (Token $token): TokenKind => $token->kind, $tokens));
67 }
68
69 public function testSkipTrivia(): void
70 {
71 $scanner = new Scanner();
72 $cursor = new Cursor(" \t// line\n\t/* block\n */\r\n , next");
73
74 $scanner->skipTrivia($cursor);
75
76 self::assertSame('next', $cursor->take(4));
77 self::assertTrue($cursor->eof());
78 }
79
80 public function testScanKeepsLineDirectivesAtTheStartOfALine(): void
81 {
82 $tokens = (new Scanner())->scan("#line 12 \"x.y\"\n%token A\n#line 4\n%%");
83
84 self::assertSame(
85 [[TokenKind::Line, '#line 12 "x.y"'], [TokenKind::Directive, 'token'], [TokenKind::Identifier, 'A'], [TokenKind::Line, '#line 4'], [TokenKind::Section, '%%'], [TokenKind::End, '']],
86 array_map(static fn (Token $token): array => [$token->kind, $token->text], $tokens),
87 );
88 }
89
90 public function testSkipTriviaRejectsAnUnterminatedComment(): void
91 {
92 $this->expectException(SyntaxException::class);
93 $this->expectExceptionMessage("Missing '*/' closing the comment opened at 1:2");
94
95 (new Scanner())->skipTrivia(new Cursor(' /* open'));
96 }
97
98 public function testNext(): void
99 {
100 $scanner = new Scanner();
101
102 self::assertSame(TokenKind::Directive, $scanner->next(new Cursor('%left'))[0]->kind);
103 self::assertSame(TokenKind::Code, $scanner->next(new Cursor('{ x }'))[0]->kind);
104 self::assertSame(TokenKind::Identifier, $scanner->next(new Cursor('.opt'))[0]->kind);
105 self::assertSame(TokenKind::Integer, $scanner->next(new Cursor('42'))[0]->kind);
106 self::assertSame(TokenKind::TranslatableString, $scanner->next(new Cursor('_("x")'))[0]->kind);
107 self::assertSame(TokenKind::Tag, $scanner->next(new Cursor('<int>'))[0]->kind);
108 self::assertSame(TokenKind::Pipe, $scanner->next(new Cursor('|'))[0]->kind);
109 }
110
111 public function testNextRejectsAnInvalidCharacter(): void
112 {
113 $this->expectException(SyntaxException::class);
114 $this->expectExceptionMessage("Invalid character: '@' at 1:1");
115
116 (new Scanner())->next(new Cursor('@'));
117 }
118
119 public function testPercent(): void
120 {
121 $scanner = new Scanner();
122 $prologue = $scanner->percent(new Cursor('%{ code %}'), new Location(1, 1));
123 $section = $scanner->percent(new Cursor('%%'), new Location(2, 1));
124 $predicate = $scanner->percent(new Cursor("%? \n {ok}"), new Location(3, 1));
125 $spelled = $scanner->percent(new Cursor('%pure_parser'), new Location(4, 1));
126 $cursor = new Cursor('%name-prefix = "yy"');
127 $option = $scanner->percent($cursor, new Location(5, 1));
128
129 self::assertSame([TokenKind::Prologue, ' code '], [$prologue->kind, $prologue->text]);
130 self::assertSame(TokenKind::Section, $section->kind);
131 self::assertSame([TokenKind::Predicate, 'ok'], [$predicate->kind, $predicate->text]);
132 self::assertSame(['pure-parser', '%pure_parser'], [$spelled->text, $spelled->raw]);
133 self::assertSame('name-prefix', $option->text);
134 self::assertSame('"yy"', $cursor->take(4));
135 }
136
137 public function testPercentRejectsAnUnknownDirective(): void
138 {
139 $this->expectException(SyntaxException::class);
140 $this->expectExceptionMessage('Invalid directive: %tokens at 1:1');
141
142 (new Scanner())->percent(new Cursor('%tokens'), new Location(1, 1));
143 }
144
145 public function testPercentRejectsABarePercentSign(): void
146 {
147 $this->expectException(SyntaxException::class);
148 $this->expectExceptionMessage('Invalid directive: % at 1:1');
149
150 (new Scanner())->percent(new Cursor('% '), new Location(1, 1));
151 }
152
153 public function testIdentifier(): void
154 {
155 $scanner = new Scanner();
156 $plain = $scanner->identifier(new Cursor('expr.opt rest'), new Location(1, 1));
157 $lhs = $scanner->identifier(new Cursor("expr /* c */ [e] \n:"), new Location(2, 1));
158 $referenced = $scanner->identifier(new Cursor('expr[e] NUM'), new Location(3, 1));
159
160 self::assertSame([TokenKind::Identifier, 'expr.opt'], [$plain[0]->kind, $plain[0]->text]);
161 self::assertCount(1, $plain);
162 self::assertSame([TokenKind::IdentifierColon, 'expr'], [$lhs[0]->kind, $lhs[0]->text]);
163 self::assertSame([TokenKind::BracketedIdentifier, 'e', '1:14'], [$lhs[1]->kind, $lhs[1]->text, (string) $lhs[1]->location]);
164 self::assertSame([TokenKind::Identifier, TokenKind::BracketedIdentifier], [$referenced[0]->kind, $referenced[1]->kind]);
165 }
166
167 public function testBracketed(): void
168 {
169 $cursor = new Cursor('[ name ] rest');
170
171 $token = (new Scanner())->bracketed($cursor);
172
173 self::assertSame([TokenKind::BracketedIdentifier, 'name', '1:1'], [$token->kind, $token->text, (string) $token->location]);
174 self::assertSame(' rest', $cursor->take(5));
175 }
176
177 public function testBracketedRejectsAMissingName(): void
178 {
179 $this->expectException(SyntaxException::class);
180 $this->expectExceptionMessage('An identifier expected at 1:2');
181
182 (new Scanner())->bracketed(new Cursor('[1]'));
183 }
184
185 public function testBracketedRejectsAMissingCloser(): void
186 {
187 $this->expectException(SyntaxException::class);
188 $this->expectExceptionMessage("Expected ']' but found ';' at 1:6");
189
190 (new Scanner())->bracketed(new Cursor('[name;'));
191 }
192
193 public function testBracketedRejectsTheEndOfTheFile(): void
194 {
195 $this->expectException(SyntaxException::class);
196 $this->expectExceptionMessage("Expected ']' but found end of file at 1:6");
197
198 (new Scanner())->bracketed(new Cursor('[name'));
199 }
200
201 public function testInteger(): void
202 {
203 $scanner = new Scanner();
204 $decimal = $scanner->integer(new Cursor('258 '), new Location(1, 1));
205 $hex = $scanner->integer(new Cursor('0x1F;'), new Location(1, 5));
206
207 self::assertSame([TokenKind::Integer, '258', '258'], [$decimal->kind, $decimal->text, $decimal->raw]);
208 self::assertSame(['31', '0x1F'], [$hex->text, $hex->raw]);
209 self::assertSame('2147483647', $scanner->integer(new Cursor('2147483647'), new Location(1, 1))->text);
210 self::assertSame('2147483647', $scanner->integer(new Cursor('0x000000007FFFFFFF'), new Location(1, 1))->text);
211 self::assertSame('0', $scanner->integer(new Cursor('0x0'), new Location(1, 1))->text);
212 }
213
214 public function testIntegerRejectsADecimalOutOfRange(): void
215 {
216 $this->expectException(SyntaxException::class);
217 $this->expectExceptionMessage("Integer out of range: '2147483648' at 2:5");
218
219 (new Scanner())->integer(new Cursor('2147483648'), new Location(2, 5));
220 }
221
222 public function testIntegerRejectsADecimalTooLongToRead(): void
223 {
224 $this->expectException(SyntaxException::class);
225 $this->expectExceptionMessage("Integer out of range: '999999999999999999999' at 2:5");
226
227 (new Scanner())->integer(new Cursor('999999999999999999999'), new Location(2, 5));
228 }
229
230 public function testIntegerRejectsAHexadecimalOutOfRange(): void
231 {
232 $this->expectException(SyntaxException::class);
233 $this->expectExceptionMessage("Integer out of range: '0x80000000' at 2:5");
234
235 (new Scanner())->integer(new Cursor('0x80000000'), new Location(2, 5));
236 }
237
238 public function testIntegerRejectsAHexadecimalTooLongToRead(): void
239 {
240 $this->expectException(SyntaxException::class);
241 $this->expectExceptionMessage("Integer out of range: '0xFFFFFFFFFFFFFFFFFFF' at 2:5");
242
243 (new Scanner())->integer(new Cursor('0xFFFFFFFFFFFFFFFFFFF'), new Location(2, 5));
244 }
245
246 public function testIntegerRejectsAnIdentifierStartingWithADigit(): void
247 {
248 $this->expectException(SyntaxException::class);
249 $this->expectExceptionMessage('Invalid identifier: 1st_rule at 1:1');
250
251 (new Scanner())->integer(new Cursor('1st_rule:'), new Location(1, 1));
252 }
253
254 public function testLiteral(): void
255 {
256 $scanner = new Scanner();
257 $char = $scanner->literal(new Cursor("'\\n'"), new Location(1, 1));
258 $string = $scanner->literal(new Cursor('"a\"b"'), new Location(1, 1));
259 $translatable = $scanner->literal(new Cursor('_("number")'), new Location(1, 1));
260 $tag = $scanner->literal(new Cursor('<int>'), new Location(1, 1));
261
262 self::assertSame([TokenKind::CharLiteral, "\n", "'\\n'"], [$char?->kind, $char?->text, $char?->raw]);
263 self::assertSame([TokenKind::String, 'a"b', '"a\\"b"'], [$string?->kind, $string?->text, $string?->raw]);
264 self::assertSame([TokenKind::TranslatableString, 'number', '_("number")'], [$translatable?->kind, $translatable?->text, $translatable?->raw]);
265 self::assertSame([TokenKind::Tag, 'int'], [$tag?->kind, $tag?->text]);
266 self::assertNull($scanner->literal(new Cursor(':'), new Location(1, 1)));
267 }
268
269 public function testLiteralRejectsAnUnterminatedCharacter(): void
270 {
271 $this->expectException(SyntaxException::class);
272 $this->expectExceptionMessage("Missing ''' closing the character literal opened at 1:1");
273
274 (new Scanner())->literal(new Cursor("'a\n'"), new Location(1, 1));
275 }
276
277 public function testLiteralRejectsAnEmptyCharacter(): void
278 {
279 $this->expectException(SyntaxException::class);
280 $this->expectExceptionMessage('Empty character literal at 1:1');
281
282 (new Scanner())->literal(new Cursor("''"), new Location(1, 1));
283 }
284
285 public function testLiteralRejectsAWideCharacter(): void
286 {
287 $this->expectException(SyntaxException::class);
288 $this->expectExceptionMessage('Extra characters in character literal at 1:1');
289
290 (new Scanner())->literal(new Cursor("'ab'"), new Location(1, 1));
291 }
292
293 public function testLiteralRejectsAnUnterminatedString(): void
294 {
295 $this->expectException(SyntaxException::class);
296 $this->expectExceptionMessage("Missing '\"' closing the string opened at 1:1");
297
298 (new Scanner())->literal(new Cursor("\"abc\n"), new Location(1, 1));
299 }
300
301 public function testTag(): void
302 {
303 $scanner = new Scanner();
304 $any = $scanner->tag(new Cursor('<*>'), new Location(1, 1));
305 $none = $scanner->tag(new Cursor('<>'), new Location(1, 1));
306 $nested = $scanner->tag(new Cursor('<std::map<int, std::vector<x>>*>'), new Location(1, 1));
307 $arrow = $scanner->tag(new Cursor('<a->b>'), new Location(1, 1));
308
309 self::assertSame([TokenKind::TagAny, '*'], [$any->kind, $any->text]);
310 self::assertSame([TokenKind::TagNone, ''], [$none->kind, $none->text]);
311 self::assertSame([TokenKind::Tag, 'std::map<int, std::vector<x>>*'], [$nested->kind, $nested->text]);
312 self::assertSame('a->b', $arrow->text);
313 }
314
315 public function testTagRejectsAnUnterminatedTag(): void
316 {
317 $this->expectException(SyntaxException::class);
318 $this->expectExceptionMessage("Missing '>' closing the tag opened at 1:1");
319
320 (new Scanner())->tag(new Cursor('<int'), new Location(1, 1));
321 }
322
323 public function testPunctuation(): void
324 {
325 $scanner = new Scanner();
326
327 self::assertSame(TokenKind::Colon, $scanner->punctuation(new Cursor(':'), new Location(1, 1))?->kind);
328 self::assertSame(TokenKind::Equal, $scanner->punctuation(new Cursor('='), new Location(1, 1))?->kind);
329 self::assertSame(TokenKind::Pipe, $scanner->punctuation(new Cursor('|'), new Location(1, 1))?->kind);
330 self::assertSame(TokenKind::Semicolon, $scanner->punctuation(new Cursor(';'), new Location(1, 1))?->kind);
331 self::assertSame(TokenKind::BracketedIdentifier, $scanner->punctuation(new Cursor('[x]'), new Location(1, 1))?->kind);
332 self::assertNull($scanner->punctuation(new Cursor('@'), new Location(1, 1)));
333 }
334}
335