packages/sql-faker/tests/Unit/Compiler/Bison/Lexer/BisonTokenStreamTest.php

1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\SqlFaker\Compiler\Bison\Lexer;
6
7use PHPUnit\Framework\Attributes\CoversClass;
8use PHPUnit\Framework\Attributes\DataProvider;
9use PHPUnit\Framework\Attributes\UsesClass;
10use PHPUnit\Framework\TestCase;
11use SqlFaker\Compiler\Bison\Lexer\ActionScanner;
12use SqlFaker\Compiler\Bison\Lexer\BisonLexeme;
13use SqlFaker\Compiler\Bison\Lexer\BisonLexer;
14use SqlFaker\Compiler\Bison\Lexer\BisonScannerChain;
15use SqlFaker\Compiler\Bison\Lexer\BisonToken;
16use SqlFaker\Compiler\Bison\Lexer\BisonTokenStream;
17use SqlFaker\Compiler\Bison\Lexer\BisonTrivia;
18use SqlFaker\Compiler\Bison\Lexer\DirectiveScanner;
19use SqlFaker\Compiler\Bison\Lexer\IdentifierScanner;
20use SqlFaker\Compiler\Bison\Lexer\NumberScanner;
21use SqlFaker\Compiler\Bison\Lexer\PunctuationScanner;
22use SqlFaker\Compiler\Bison\Lexer\QuotedLiteralScanner;
23use SqlFaker\Compiler\Bison\Lexer\SourceCursor;
24use SqlFaker\Compiler\Bison\Lexer\TypeTagScanner;
25use SqlFaker\Compiler\GrammarParseException;
26
27#[CoversClass(BisonTokenStream::class)]
28#[UsesClass(ActionScanner::class)]
29#[UsesClass(BisonLexeme::class)]
30#[UsesClass(BisonLexer::class)]
31#[UsesClass(BisonScannerChain::class)]
32#[UsesClass(BisonToken::class)]
33#[UsesClass(BisonTrivia::class)]
34#[UsesClass(DirectiveScanner::class)]
35#[UsesClass(GrammarParseException::class)]
36#[UsesClass(IdentifierScanner::class)]
37#[UsesClass(NumberScanner::class)]
38#[UsesClass(PunctuationScanner::class)]
39#[UsesClass(QuotedLiteralScanner::class)]
40#[UsesClass(SourceCursor::class)]
41#[UsesClass(TypeTagScanner::class)]
42final class BisonTokenStreamTest extends TestCase
43{
44    public function testNextReturnsEofForEmptyInput(): void
45    {
46        $lexer = BisonTokenStream::over('');
47
48        $token = $lexer->next();
49
50        self::assertSame(BisonLexeme::Eof, $token->type);
51        self::assertSame('', $token->value);
52        self::assertSame(0, $token->offset);
53    }
54
55    public function testNextReturnsEofForWhitespaceOnlyInput(): void
56    {
57        $lexer = BisonTokenStream::over("   \t\n\r  ");
58
59        $token = $lexer->next();
60
61        self::assertSame(BisonLexeme::Eof, $token->type);
62    }
63
64    public function testNextReturnsEofAfterEof(): void
65    {
66        $lexer = BisonTokenStream::over('x');
67
68        $lexer->next();
69        $eof1 = $lexer->next();
70        $eof2 = $lexer->next();
71
72        self::assertSame(BisonLexeme::Eof, $eof1->type);
73        self::assertSame(BisonLexeme::Eof, $eof2->type);
74    }
75
76    public function testNextReturnsTokenFromBuffer(): void
77    {
78        $lexer = BisonTokenStream::over('foo bar');
79
80        $peeked = $lexer->peek();
81        $token = $lexer->next();
82
83        self::assertSame($peeked->type, $token->type);
84        self::assertSame($peeked->value, $token->value);
85    }
86
87
88    public function testNextSkipsWhitespace(): void
89    {
90        $lexer = BisonTokenStream::over("   \t\n\r  foo");
91
92        $token = $lexer->next();
93
94        self::assertSame(BisonLexeme::Identifier, $token->type);
95        self::assertSame('foo', $token->value);
96    }
97
98    public function testNextSkipsLineComment(): void
99    {
100        $lexer = BisonTokenStream::over("// this is a comment\nfoo");
101
102        $token = $lexer->next();
103
104        self::assertSame(BisonLexeme::Identifier, $token->type);
105        self::assertSame('foo', $token->value);
106    }
107
108    public function testNextSkipsLineCommentAtEof(): void
109    {
110        $lexer = BisonTokenStream::over('// comment without newline');
111
112        $token = $lexer->next();
113
114        self::assertSame(BisonLexeme::Eof, $token->type);
115    }
116
117    public function testNextSkipsBlockComment(): void
118    {
119        $lexer = BisonTokenStream::over('/* block comment */ foo');
120
121        $token = $lexer->next();
122
123        self::assertSame(BisonLexeme::Identifier, $token->type);
124        self::assertSame('foo', $token->value);
125    }
126
127    public function testNextSkipsBlockCommentMultiline(): void
128    {
129        $input = <<<'INPUT'
130/* line 1
131   line 2
132   line 3 */
133foo
134INPUT;
135        $lexer = BisonTokenStream::over($input);
136
137        $token = $lexer->next();
138
139        self::assertSame(BisonLexeme::Identifier, $token->type);
140        self::assertSame('foo', $token->value);
141    }
142
143    public function testNextSkipsBlockCommentUnterminated(): void
144    {
145        $lexer = BisonTokenStream::over('/* unterminated');
146
147        $token = $lexer->next();
148
149        self::assertSame(BisonLexeme::Eof, $token->type);
150    }
151
152    public function testNextSkipsConsecutiveLineComments(): void
153    {
154        $lexer = BisonTokenStream::over("// first\n// second\nfoo");
155
156        $token = $lexer->next();
157
158        self::assertSame(BisonLexeme::Identifier, $token->type);
159        self::assertSame('foo', $token->value);
160    }
161
162    public function testNextSkipsConsecutiveBlockComments(): void
163    {
164        $lexer = BisonTokenStream::over('/* first */ /* second */ foo');
165
166        $token = $lexer->next();
167
168        self::assertSame(BisonLexeme::Identifier, $token->type);
169        self::assertSame('foo', $token->value);
170    }
171
172    public function testNextSkipsMixedComments(): void
173    {
174        $lexer = BisonTokenStream::over("/* block */ // line\nfoo");
175
176        $token = $lexer->next();
177
178        self::assertSame(BisonLexeme::Identifier, $token->type);
179        self::assertSame('foo', $token->value);
180    }
181
182
183    public function testNextIdentifier(): void
184    {
185        $lexer = BisonTokenStream::over('simple_ident');
186
187        $token = $lexer->next();
188
189        self::assertSame(BisonLexeme::Identifier, $token->type);
190        self::assertSame('simple_ident', $token->value);
191        self::assertSame(0, $token->offset);
192    }
193
194    public function testNextIdentifierSingleChar(): void
195    {
196        $lexer = BisonTokenStream::over('a');
197
198        $token = $lexer->next();
199
200        self::assertSame(BisonLexeme::Identifier, $token->type);
201        self::assertSame('a', $token->value);
202    }
203
204    public function testNextIdentifierWithDigits(): void
205    {
206        $lexer = BisonTokenStream::over('foo123');
207
208        $token = $lexer->next();
209
210        self::assertSame(BisonLexeme::Identifier, $token->type);
211        self::assertSame('foo123', $token->value);
212    }
213
214    public function testNextIdentifierWithDots(): void
215    {
216        $lexer = BisonTokenStream::over('api.pure.full');
217
218        $token = $lexer->next();
219
220        self::assertSame(BisonLexeme::Identifier, $token->type);
221        self::assertSame('api.pure.full', $token->value);
222    }
223
224    public function testNextIdentifierWithUnderscore(): void
225    {
226        $lexer = BisonTokenStream::over('_private');
227
228        $token = $lexer->next();
229
230        self::assertSame(BisonLexeme::Identifier, $token->type);
231        self::assertSame('_private', $token->value);
232    }
233
234    public function testNextIdentifierUpperCase(): void
235    {
236        $lexer = BisonTokenStream::over('SELECT_SYM');
237
238        $token = $lexer->next();
239
240        self::assertSame(BisonLexeme::Identifier, $token->type);
241        self::assertSame('SELECT_SYM', $token->value);
242    }
243
244
245    public function testNextNumber(): void
246    {
247        $lexer = BisonTokenStream::over('12345');
248
249        $token = $lexer->next();
250
251        self::assertSame(BisonLexeme::Number, $token->type);
252        self::assertSame(12345, $token->value);
253        self::assertSame(0, $token->offset);
254    }
255
256    public function testNextNumberSingleDigit(): void
257    {
258        $lexer = BisonTokenStream::over('7');
259
260        $token = $lexer->next();
261
262        self::assertSame(BisonLexeme::Number, $token->type);
263        self::assertSame(7, $token->value);
264    }
265
266    public function testNextNumberZero(): void
267    {
268        $lexer = BisonTokenStream::over('0');
269
270        $token = $lexer->next();
271
272        self::assertSame(BisonLexeme::Number, $token->type);
273        self::assertSame(0, $token->value);
274    }
275
276
277    public function testNextStringLiteral(): void
278    {
279        $lexer = BisonTokenStream::over('"hello world"');
280
281        $token = $lexer->next();
282
283        self::assertSame(BisonLexeme::StringLiteral, $token->type);
284        self::assertSame('hello world', $token->value);
285    }
286
287    public function testNextStringLiteralEmpty(): void
288    {
289        $lexer = BisonTokenStream::over('""');
290
291        $token = $lexer->next();
292
293        self::assertSame(BisonLexeme::StringLiteral, $token->type);
294        self::assertSame('', $token->value);
295    }
296
297    public function testNextStringLiteralWithEscapedQuote(): void
298    {
299        $lexer = BisonTokenStream::over('"say \"hi\""');
300
301        $token = $lexer->next();
302
303        self::assertSame(BisonLexeme::StringLiteral, $token->type);
304        self::assertSame('say "hi"', $token->value);
305    }
306
307    public function testNextStringLiteralWithBackslash(): void
308    {
309        $lexer = BisonTokenStream::over('"path\\\\to\\\\file"');
310
311        $token = $lexer->next();
312
313        self::assertSame(BisonLexeme::StringLiteral, $token->type);
314        self::assertSame('path\\to\\file', $token->value);
315    }
316
317    public function testNextStringLiteralUnterminated(): void
318    {
319        $lexer = BisonTokenStream::over('"unterminated');
320
321        $token = $lexer->next();
322
323        self::assertSame(BisonLexeme::StringLiteral, $token->type);
324        self::assertSame('unterminated', $token->value);
325    }
326
327    public function testNextStringLiteralEscapeAtEnd(): void
328    {
329        $lexer = BisonTokenStream::over('"end\\');
330
331        $token = $lexer->next();
332
333        self::assertSame(BisonLexeme::StringLiteral, $token->type);
334        self::assertSame('end', $token->value);
335    }
336
337
338    public function testNextCharLiteral(): void
339    {
340        $lexer = BisonTokenStream::over("'c'");
341
342        $token = $lexer->next();
343
344        self::assertSame(BisonLexeme::CharLiteral, $token->type);
345        self::assertSame('c', $token->value);
346    }
347
348    public function testNextCharLiteralEmpty(): void
349    {
350        $lexer = BisonTokenStream::over("''");
351
352        $token = $lexer->next();
353
354        self::assertSame(BisonLexeme::CharLiteral, $token->type);
355        self::assertSame('', $token->value);
356    }
357
358    public function testNextCharLiteralWithEscape(): void
359    {
360        $lexer = BisonTokenStream::over("'\\n'");
361
362        $token = $lexer->next();
363
364        self::assertSame(BisonLexeme::CharLiteral, $token->type);
365        self::assertSame('n', $token->value);
366    }
367
368    public function testNextCharLiteralUnterminated(): void
369    {
370        $lexer = BisonTokenStream::over("'x");
371
372        $token = $lexer->next();
373
374        self::assertSame(BisonLexeme::CharLiteral, $token->type);
375        self::assertSame('x', $token->value);
376    }
377
378
379    public function testNextColon(): void
380    {
381        $lexer = BisonTokenStream::over(':');
382
383        $token = $lexer->next();
384
385        self::assertSame(BisonLexeme::Colon, $token->type);
386        self::assertSame(':', $token->value);
387    }
388
389    public function testNextSemicolon(): void
390    {
391        $lexer = BisonTokenStream::over(';');
392
393        $token = $lexer->next();
394
395        self::assertSame(BisonLexeme::Semicolon, $token->type);
396        self::assertSame(';', $token->value);
397    }
398
399    public function testNextPipe(): void
400    {
401        $lexer = BisonTokenStream::over('|');
402
403        $token = $lexer->next();
404
405        self::assertSame(BisonLexeme::Pipe, $token->type);
406        self::assertSame('|', $token->value);
407    }
408
409    public function testNextPercentPercent(): void
410    {
411        $lexer = BisonTokenStream::over('%%');
412
413        $token = $lexer->next();
414
415        self::assertSame(BisonLexeme::PercentPercent, $token->type);
416        self::assertSame('%%', $token->value);
417    }
418
419
420    public function testNextDirective(): void
421    {
422        $lexer = BisonTokenStream::over('%token');
423
424        $token = $lexer->next();
425
426        self::assertSame(BisonLexeme::Directive, $token->type);
427        self::assertSame('%token', $token->value);
428    }
429
430    public function testNextDirectiveWithHyphen(): void
431    {
432        $lexer = BisonTokenStream::over('%parse-param');
433
434        $token = $lexer->next();
435
436        self::assertSame(BisonLexeme::Directive, $token->type);
437        self::assertSame('%parse-param', $token->value);
438    }
439
440    public function testNextDirectiveWithNumbers(): void
441    {
442        $lexer = BisonTokenStream::over('%define123');
443
444        $token = $lexer->next();
445
446        self::assertSame(BisonLexeme::Directive, $token->type);
447        self::assertSame('%define123', $token->value);
448    }
449
450    public function testNextDirectiveWithDots(): void
451    {
452        $lexer = BisonTokenStream::over('%api.pure');
453
454        $token = $lexer->next();
455
456        self::assertSame(BisonLexeme::Directive, $token->type);
457        self::assertSame('%api.pure', $token->value);
458    }
459
460
461    public function testNextTypeTag(): void
462    {
463        $lexer = BisonTokenStream::over('<node_type>');
464
465        $token = $lexer->next();
466
467        self::assertSame(BisonLexeme::TypeTag, $token->type);
468        self::assertSame('node_type', $token->value);
469    }
470
471    public function testNextTypeTagEmpty(): void
472    {
473        $lexer = BisonTokenStream::over('<>');
474
475        $token = $lexer->next();
476
477        self::assertSame(BisonLexeme::TypeTag, $token->type);
478        self::assertSame('', $token->value);
479    }
480
481    public function testNextTypeTagTrimmed(): void
482    {
483        $lexer = BisonTokenStream::over('<  spaced  >');
484
485        $token = $lexer->next();
486
487        self::assertSame(BisonLexeme::TypeTag, $token->type);
488        self::assertSame('spaced', $token->value);
489    }
490
491    public function testNextTypeTagWithSpecialChars(): void
492    {
493        $lexer = BisonTokenStream::over('<Item*>');
494
495        $token = $lexer->next();
496
497        self::assertSame(BisonLexeme::TypeTag, $token->type);
498        self::assertSame('Item*', $token->value);
499    }
500
501
502    public function testNextPrologue(): void
503    {
504        $lexer = BisonTokenStream::over('%{ #include <stdio.h> %}');
505
506        $token = $lexer->next();
507
508        self::assertSame(BisonLexeme::Prologue, $token->type);
509        self::assertSame(' #include <stdio.h> ', $token->value);
510    }
511
512    public function testNextPrologueEmpty(): void
513    {
514        $lexer = BisonTokenStream::over('%{%}');
515
516        $token = $lexer->next();
517
518        self::assertSame(BisonLexeme::Prologue, $token->type);
519        self::assertSame('', $token->value);
520    }
521
522    public function testNextPrologueWithWhitespaceOnly(): void
523    {
524        $lexer = BisonTokenStream::over('%{   %}');
525
526        $token = $lexer->next();
527
528        self::assertSame(BisonLexeme::Prologue, $token->type);
529        self::assertSame('   ', $token->value);
530    }
531
532    public function testNextPrologueUnterminated(): void
533    {
534        $lexer = BisonTokenStream::over('%{ incomplete prologue');
535
536        $token = $lexer->next();
537
538        self::assertSame(BisonLexeme::Prologue, $token->type);
539        self::assertSame(' incomplete prologue', $token->value);
540    }
541
542
543    public function testNextAction(): void
544    {
545        $lexer = BisonTokenStream::over('{ $$ = $1; }');
546
547        $token = $lexer->next();
548
549        self::assertSame(BisonLexeme::Action, $token->type);
550        self::assertSame(' $$ = $1; ', $token->value);
551    }
552
553    public function testNextActionEmpty(): void
554    {
555        $lexer = BisonTokenStream::over('{}');
556
557        $token = $lexer->next();
558
559        self::assertSame(BisonLexeme::Action, $token->type);
560        self::assertSame('', $token->value);
561    }
562
563    public function testNextActionWithWhitespaceOnly(): void
564    {
565        $lexer = BisonTokenStream::over('{   }');
566
567        $token = $lexer->next();
568
569        self::assertSame(BisonLexeme::Action, $token->type);
570        self::assertSame('   ', $token->value);
571    }
572
573    public function testNextActionNested(): void
574    {
575        $lexer = BisonTokenStream::over('{ if (x) { y(); } }');
576
577        $token = $lexer->next();
578
579        self::assertSame(BisonLexeme::Action, $token->type);
580        self::assertSame(' if (x) { y(); } ', $token->value);
581    }
582
583    public function testNextActionDeeplyNested(): void
584    {
585        $lexer = BisonTokenStream::over('{ a { b { c } d } e }');
586
587        $token = $lexer->next();
588
589        self::assertSame(BisonLexeme::Action, $token->type);
590        self::assertSame(' a { b { c } d } e ', $token->value);
591    }
592
593    public function testNextActionWithStringContainingBrace(): void
594    {
595        $lexer = BisonTokenStream::over('{ printf("{"); }');
596
597        $token = $lexer->next();
598
599        self::assertSame(BisonLexeme::Action, $token->type);
600        self::assertSame(' printf("{"); ', $token->value);
601    }
602
603    public function testNextActionWithCharLiteralContainingBrace(): void
604    {
605        $lexer = BisonTokenStream::over("{ char c = '{'; }");
606
607        $token = $lexer->next();
608
609        self::assertSame(BisonLexeme::Action, $token->type);
610        self::assertSame(" char c = '{'; ", $token->value);
611    }
612
613    public function testNextActionWithLineComment(): void
614    {
615        $lexer = BisonTokenStream::over("{ x = 1; // comment\n y = 2; }");
616
617        $token = $lexer->next();
618
619        self::assertSame(BisonLexeme::Action, $token->type);
620        self::assertIsString($token->value);
621        self::assertStringContainsString('x = 1;', $token->value);
622        self::assertStringContainsString('y = 2;', $token->value);
623    }
624
625    public function testNextActionWithBlockComment(): void
626    {
627        $lexer = BisonTokenStream::over('{ x = 1; /* { not a brace } */ y = 2; }');
628
629        $token = $lexer->next();
630
631        self::assertSame(BisonLexeme::Action, $token->type);
632        self::assertIsString($token->value);
633        self::assertStringContainsString('x = 1;', $token->value);
634        self::assertStringContainsString('y = 2;', $token->value);
635    }
636
637    public function testNextActionUnterminated(): void
638    {
639        $lexer = BisonTokenStream::over('{ incomplete action');
640
641        $token = $lexer->next();
642
643        self::assertSame(BisonLexeme::Action, $token->type);
644        self::assertSame(' incomplete action', $token->value);
645    }
646
647    public function testNextActionWithUnterminatedString(): void
648    {
649        $lexer = BisonTokenStream::over('{ "unterminated }');
650
651        $token = $lexer->next();
652
653        self::assertSame(BisonLexeme::Action, $token->type);
654        self::assertIsString($token->value);
655    }
656
657
658    public function testNextMultipleTokens(): void
659    {
660        $lexer = BisonTokenStream::over('%token FOO 123 "alias"');
661
662        $t1 = $lexer->next();
663        $t2 = $lexer->next();
664        $t3 = $lexer->next();
665        $t4 = $lexer->next();
666
667        self::assertSame(BisonLexeme::Directive, $t1->type);
668        self::assertSame('%token', $t1->value);
669
670        self::assertSame(BisonLexeme::Identifier, $t2->type);
671        self::assertSame('FOO', $t2->value);
672
673        self::assertSame(BisonLexeme::Number, $t3->type);
674        self::assertSame(123, $t3->value);
675
676        self::assertSame(BisonLexeme::StringLiteral, $t4->type);
677        self::assertSame('alias', $t4->value);
678    }
679
680    public function testNextTracksOffset(): void
681    {
682        $lexer = BisonTokenStream::over('foo  bar');
683
684        $t1 = $lexer->next();
685        $t2 = $lexer->next();
686
687        self::assertSame(0, $t1->offset);
688        self::assertSame(5, $t2->offset);
689    }
690
691    public function testNextTracksOffsetAfterWhitespace(): void
692    {
693        $lexer = BisonTokenStream::over('   foo');
694
695        $token = $lexer->next();
696
697        self::assertSame(3, $token->offset);
698    }
699
700    public function testNextTracksOffsetAfterComment(): void
701    {
702        $lexer = BisonTokenStream::over('/* x */ foo');
703
704        $token = $lexer->next();
705
706        self::assertSame(8, $token->offset);
707    }
708
709
710    public function testNextThrowsForUnexpectedCharacter(): void
711    {
712        $lexer = BisonTokenStream::over('@');
713
714        $this->expectException(GrammarParseException::class);
715        $this->expectExceptionMessage("Unexpected character '@' at offset 0");
716
717        $lexer->next();
718    }
719
720    public function testNextThrowsForUnexpectedSlash(): void
721    {
722        $lexer = BisonTokenStream::over('/ not a comment');
723
724        $this->expectException(GrammarParseException::class);
725        $this->expectExceptionMessage("Unexpected '/' at offset 0");
726
727        $lexer->next();
728    }
729
730    public function testNextThrowsForEmptyDirective(): void
731    {
732        $lexer = BisonTokenStream::over('% ');
733
734        $this->expectException(GrammarParseException::class);
735        $this->expectExceptionMessage("Unexpected '%' at offset 0");
736
737        $lexer->next();
738    }
739
740    public function testNextThrowsForUnterminatedTypeTag(): void
741    {
742        $lexer = BisonTokenStream::over('<unterminated');
743
744        $this->expectException(GrammarParseException::class);
745        $this->expectExceptionMessage('Unterminated type tag starting at offset 0');
746
747        $lexer->next();
748    }
749
750    public function testNextThrowsForDigitStartingIdentifierPosition(): void
751    {
752        $lexer = BisonTokenStream::over('123abc');
753
754        $t1 = $lexer->next();
755        $t2 = $lexer->next();
756
757        self::assertSame(BisonLexeme::Number, $t1->type);
758        self::assertSame(123, $t1->value);
759        self::assertSame(BisonLexeme::Identifier, $t2->type);
760        self::assertSame('abc', $t2->value);
761    }
762
763
764    public function testNextCompleteGrammarFragment(): void
765    {
766        $input = <<<'YY'
767%{
768#include <stdio.h>
769%}
770%start expr
771%token <num> NUMBER 100
772%left PLUS MINUS
773%%
774expr:
775      NUMBER
776    | expr PLUS expr { $$ = $1 + $3; }
777;
778YY;
779
780        $lexer = BisonTokenStream::over($input);
781        $t0 = $lexer->next();
782        $t1 = $lexer->next();
783        $t2 = $lexer->next();
784        $t3 = $lexer->next();
785        $t4 = $lexer->next();
786        $t5 = $lexer->next();
787        $t6 = $lexer->next();
788        $t7 = $lexer->next();
789        $t8 = $lexer->next();
790        $t9 = $lexer->next();
791        $t10 = $lexer->next();
792        $t11 = $lexer->next();
793        $t12 = $lexer->next();
794        $t13 = $lexer->next();
795        $t14 = $lexer->next();
796        $t15 = $lexer->next();
797        $t16 = $lexer->next();
798        $t17 = $lexer->next();
799        $t18 = $lexer->next();
800        $t19 = $lexer->next();
801        $t20 = $lexer->next();
802
803        self::assertSame(BisonLexeme::Prologue, $t0->type);
804        self::assertSame(BisonLexeme::Directive, $t1->type);
805        self::assertSame(BisonLexeme::Identifier, $t2->type);
806        self::assertSame(BisonLexeme::Directive, $t3->type);
807        self::assertSame(BisonLexeme::TypeTag, $t4->type);
808        self::assertSame(BisonLexeme::Identifier, $t5->type);
809        self::assertSame(BisonLexeme::Number, $t6->type);
810        self::assertSame(BisonLexeme::Directive, $t7->type);
811        self::assertSame(BisonLexeme::Identifier, $t8->type);
812        self::assertSame(BisonLexeme::Identifier, $t9->type);
813        self::assertSame(BisonLexeme::PercentPercent, $t10->type);
814        self::assertSame(BisonLexeme::Identifier, $t11->type);
815        self::assertSame(BisonLexeme::Colon, $t12->type);
816        self::assertSame(BisonLexeme::Identifier, $t13->type);
817        self::assertSame(BisonLexeme::Pipe, $t14->type);
818        self::assertSame(BisonLexeme::Identifier, $t15->type);
819        self::assertSame(BisonLexeme::Identifier, $t16->type);
820        self::assertSame(BisonLexeme::Identifier, $t17->type);
821        self::assertSame(BisonLexeme::Action, $t18->type);
822        self::assertSame(BisonLexeme::Semicolon, $t19->type);
823        self::assertSame(BisonLexeme::Eof, $t20->type);
824    }
825
826    #[DataProvider('providerNextTokenTypes')]
827    public function testNextTokenTypes(string $input, BisonLexeme $expectedType, string|int $expectedValue): void
828    {
829        $lexer = BisonTokenStream::over($input);
830
831        $token = $lexer->next();
832
833        self::assertSame($expectedType, $token->type);
834        self::assertSame($expectedValue, $token->value);
835    }
836
837    public function testPeekReturnsSameToken(): void
838    {
839        $lexer = BisonTokenStream::over('identifier');
840
841        $first = $lexer->peek();
842        $second = $lexer->peek();
843
844        self::assertSame($first->type, $second->type);
845        self::assertSame($first->value, $second->value);
846        self::assertSame($first->offset, $second->offset);
847    }
848
849    public function testPeekDoesNotConsume(): void
850    {
851        $lexer = BisonTokenStream::over('foo bar');
852
853        $peeked = $lexer->peek();
854        $next = $lexer->next();
855
856        self::assertSame($peeked->value, $next->value);
857        self::assertSame('foo', $next->value);
858    }
859
860    public function testPeekThenNextThenPeek(): void
861    {
862        $lexer = BisonTokenStream::over('foo bar baz');
863
864        $peek1 = $lexer->peek();
865        self::assertSame('foo', $peek1->value);
866
867        $next1 = $lexer->next();
868        self::assertSame('foo', $next1->value);
869
870        $peek2 = $lexer->peek();
871        self::assertSame('bar', $peek2->value);
872    }
873
874
875    public function testPeekNReturnsNthToken(): void
876    {
877        $lexer = BisonTokenStream::over('foo bar baz');
878
879        $first = $lexer->peekN(1);
880        $second = $lexer->peekN(2);
881        $third = $lexer->peekN(3);
882
883        self::assertSame('foo', $first->value);
884        self::assertSame('bar', $second->value);
885        self::assertSame('baz', $third->value);
886    }
887
888    public function testPeekNReturnsSameResult(): void
889    {
890        $lexer = BisonTokenStream::over('a b c');
891
892        $first1 = $lexer->peekN(1);
893        $second1 = $lexer->peekN(2);
894        $first2 = $lexer->peekN(1);
895        $second2 = $lexer->peekN(2);
896
897        self::assertSame($first1->value, $first2->value);
898        self::assertSame($second1->value, $second2->value);
899    }
900
901    public function testPeekNAfterPartialConsumption(): void
902    {
903        $lexer = BisonTokenStream::over('a b c d');
904
905        $lexer->peekN(3);
906
907        $lexer->next();
908
909        $peek1 = $lexer->peekN(1);
910        self::assertSame('b', $peek1->value);
911
912        $peek2 = $lexer->peekN(2);
913        self::assertSame('c', $peek2->value);
914    }
915
916    public function testPeekNThenPeekNSmaller(): void
917    {
918        $lexer = BisonTokenStream::over('a b c');
919
920        $third = $lexer->peekN(3);
921        self::assertSame('c', $third->value);
922
923        $first = $lexer->peekN(1);
924        self::assertSame('a', $first->value);
925    }
926
927    public function testNextIfConsumesATokenOfAnAcceptedKind(): void
928    {
929        $stream = BisonTokenStream::over('foo bar');
930
931        self::assertSame('foo', $stream->nextIf(BisonLexeme::Identifier)?->value);
932        self::assertSame('bar', $stream->next()->value);
933    }
934
935    public function testNextIfAcceptsAnyOfSeveralKinds(): void
936    {
937        $stream = BisonTokenStream::over('42');
938
939        self::assertSame(42, $stream->nextIf(BisonLexeme::Identifier, BisonLexeme::Number)?->value);
940    }
941
942    public function testNextIfLeavesTheStreamUnmovedWhenTheKindDoesNotMatch(): void
943    {
944        $stream = BisonTokenStream::over('foo');
945
946        self::assertNull($stream->nextIf(BisonLexeme::Number));
947        self::assertSame('foo', $stream->next()->value);
948    }
949
950    public function testNextIfAcceptsNothingWhenNoKindIsGiven(): void
951    {
952        $stream = BisonTokenStream::over('foo');
953
954        self::assertNull($stream->nextIf());
955        self::assertSame('foo', $stream->next()->value);
956    }
957    public function testOverReadsTheSourceItWasGiven(): void
958    {
959        self::assertSame('foo', BisonTokenStream::over('foo')->next()->value);
960    }
961
962    public function testNextIntReadsANumericTokenAsAnInteger(): void
963    {
964        self::assertSame(42, BisonTokenStream::over('42')->nextInt());
965    }
966
967    public function testNextIntReadsANonNumericTokenAsZero(): void
968    {
969        self::assertSame(0, BisonTokenStream::over('foo')->nextInt());
970    }
971
972    public function testConsumeRemainingTakesEverythingAfterTheTokensAlreadyRead(): void
973    {
974        $stream = BisonTokenStream::over('foo @ % <');
975
976        $stream->next();
977
978        self::assertSame(' @ % <', $stream->consumeRemaining());
979    }
980
981    public function testConsumeRemainingDiscardsBufferedLookahead(): void
982    {
983        $stream = BisonTokenStream::over('foo bar @');
984
985        $stream->peekN(2);
986
987        self::assertSame(' @', $stream->consumeRemaining());
988    }
989
990    /**
991     * @return iterable<string, array{string, BisonLexeme, string|int}>
992     */
993    public static function providerNextTokenTypes(): iterable
994    {
995        yield 'identifier' => ['foo', BisonLexeme::Identifier, 'foo'];
996        yield 'identifier with digits' => ['foo123', BisonLexeme::Identifier, 'foo123'];
997        yield 'number' => ['42', BisonLexeme::Number, 42];
998        yield 'number zero' => ['0', BisonLexeme::Number, 0];
999        yield 'string literal' => ['"str"', BisonLexeme::StringLiteral, 'str'];
1000        yield 'char literal' => ["'x'", BisonLexeme::CharLiteral, 'x'];
1001        yield 'colon' => [':', BisonLexeme::Colon, ':'];
1002        yield 'semicolon' => [';', BisonLexeme::Semicolon, ';'];
1003        yield 'pipe' => ['|', BisonLexeme::Pipe, '|'];
1004        yield 'percent percent' => ['%%', BisonLexeme::PercentPercent, '%%'];
1005        yield 'directive token' => ['%token', BisonLexeme::Directive, '%token'];
1006        yield 'directive start' => ['%start', BisonLexeme::Directive, '%start'];
1007        yield 'directive left' => ['%left', BisonLexeme::Directive, '%left'];
1008        yield 'type tag' => ['<type>', BisonLexeme::TypeTag, 'type'];
1009        yield 'prologue' => ['%{ code %}', BisonLexeme::Prologue, ' code '];
1010        yield 'action' => ['{ code }', BisonLexeme::Action, ' code '];
1011    }
1012}
1013