packages/ztd-query-core/tests/Unit/Sql/SqlTokenStreamTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Sql;
6
7use PHPUnit\Framework\Attributes\CoversClass;
8use PHPUnit\Framework\Attributes\UsesClass;
9use PHPUnit\Framework\TestCase;
10use Tests\Fake\FakeSqlLexerProfiles;
11use ZtdQuery\Sql\Profile\SqlCommentProfile;
12use ZtdQuery\Sql\Profile\SqlParameterProfile;
13use ZtdQuery\Sql\Profile\SqlQuoteProfile;
14use ZtdQuery\Sql\Profile\SqlSymbolProfile;
15use ZtdQuery\Sql\Reader\SqlBlockCommentReader;
16use ZtdQuery\Sql\Reader\SqlDelimitedReader;
17use ZtdQuery\Sql\Reader\SqlLexeme;
18use ZtdQuery\Sql\Reader\SqlParameterReader;
19use ZtdQuery\Sql\Reader\SqlTriviaReader;
20use ZtdQuery\Sql\Reader\SqlWordReader;
21use ZtdQuery\Sql\SqlLexerProfile;
22use ZtdQuery\Sql\SqlToken;
23use ZtdQuery\Sql\SqlTokenKind;
24use ZtdQuery\Sql\SqlTokenStream;
25
26#[CoversClass(SqlTokenStream::class)]
27#[UsesClass(SqlToken::class)]
28#[UsesClass(SqlLexerProfile::class)]
29#[UsesClass(SqlTokenKind::class)]
30#[UsesClass(\ZtdQuery\Sql\LexicalDelimiters::class)]
31#[UsesClass(\ZtdQuery\Sql\LexicalPattern::class)]
32#[UsesClass(\ZtdQuery\Sql\SqlIdentifierComponent::class)]
33#[UsesClass(\ZtdQuery\Sql\SqlKeywordSequence::class)]
34#[UsesClass(\ZtdQuery\Sql\SqlTokenScanner::class)]
35#[UsesClass(SqlCommentProfile::class)]
36#[UsesClass(SqlParameterProfile::class)]
37#[UsesClass(SqlQuoteProfile::class)]
38#[UsesClass(SqlSymbolProfile::class)]
39#[UsesClass(SqlBlockCommentReader::class)]
40#[UsesClass(SqlDelimitedReader::class)]
41#[UsesClass(SqlLexeme::class)]
42#[UsesClass(SqlParameterReader::class)]
43#[UsesClass(SqlTriviaReader::class)]
44#[UsesClass(SqlWordReader::class)]
45final class SqlTokenStreamTest extends TestCase
46{
47 public function testNavigatesAdjacentSignificantTokensByIdentity(): void
48 {
49 $stream = SqlTokenStream::tokenize('MATCH /* ignored */ (title)', FakeSqlLexerProfiles::standard());
50 $tokens = $stream->significantTokens();
51
52 self::assertNull($stream->significantTokenBefore($tokens[0]));
53 self::assertSame('MATCH', $stream->significantTokenBefore($tokens[1])?->text);
54 self::assertSame('(', $stream->significantTokenAfter($tokens[0])?->text);
55 self::assertNull($stream->significantTokenAfter($tokens[3]));
56 }
57
58 public function testRejectsAdjacentLookupForATokenFromAnotherStream(): void
59 {
60 $foreign = SqlTokenStream::tokenize('MATCH', FakeSqlLexerProfiles::standard())->significantTokens()[0];
61 $stream = SqlTokenStream::tokenize('MATCH(title)', FakeSqlLexerProfiles::standard());
62
63 self::assertNull($stream->significantTokenBefore($foreign));
64 self::assertNull($stream->significantTokenAfter($foreign));
65 }
66
67 public function testFindsMatchingParenthesisAcrossNestedGroups(): void
68 {
69 $stream = SqlTokenStream::tokenize('MATCH((title), body) AGAINST (?)', FakeSqlLexerProfiles::standard());
70 $tokens = $stream->significantTokens();
71 $columnsClosing = $stream->matchingClosingNestingToken($tokens[1]);
72 $queryClosing = $stream->matchingClosingNestingToken($tokens[9]);
73
74 self::assertNotNull($columnsClosing);
75 self::assertNotNull($queryClosing);
76 self::assertSame(')', $columnsClosing->text);
77 self::assertSame(19, $columnsClosing->offset);
78 self::assertSame(31, $queryClosing->offset);
79 }
80
81 public function testRejectsInvalidForeignAndUnclosedOpeningParentheses(): void
82 {
83 $stream = SqlTokenStream::tokenize('MATCH(title', FakeSqlLexerProfiles::standard());
84 $tokens = $stream->significantTokens();
85 $foreign = SqlTokenStream::tokenize('(?)', FakeSqlLexerProfiles::standard())->significantTokens()[0];
86 $closed = SqlTokenStream::tokenize('MATCH(title)', FakeSqlLexerProfiles::standard());
87 $closedTokens = $closed->significantTokens();
88 $closingSymbols = SqlTokenStream::tokenize(') )', FakeSqlLexerProfiles::standard());
89 $closingTokens = $closingSymbols->significantTokens();
90
91 self::assertNull($stream->matchingClosingNestingToken($tokens[0]));
92 self::assertNull($stream->matchingClosingNestingToken($tokens[1]));
93 self::assertNull($stream->matchingClosingNestingToken($foreign));
94 self::assertNull($closed->matchingClosingNestingToken($closedTokens[0]));
95 self::assertNull($closingSymbols->matchingClosingNestingToken($closingTokens[0]));
96 }
97
98 public function testReadsWordAndQuotedIdentifiersAtSignificantTokenOffsets(): void
99 {
100 $profile = FakeSqlLexerProfiles::allCapabilities();
101 $stream = SqlTokenStream::tokenize('plain "quo""ted" `tick``ed` [bracket name]', $profile);
102
103 self::assertSame(['name' => 'plain', 'next' => 1], $stream->identifierAt());
104 self::assertSame(['name' => 'quo"ted', 'next' => 2], $stream->identifierAt(1));
105 self::assertSame(['name' => 'tick`ed', 'next' => 3], $stream->identifierAt(2));
106 self::assertSame(['name' => 'bracket name', 'next' => 4], $stream->identifierAt(3));
107 }
108
109 public function testIdentifierAtRejectsMissingAndNonIdentifierTokens(): void
110 {
111 $standard = FakeSqlLexerProfiles::standard();
112 $allCapabilities = FakeSqlLexerProfiles::allCapabilities();
113
114 self::assertNull(SqlTokenStream::tokenize('', $standard)->identifierAt());
115 self::assertNull(SqlTokenStream::tokenize('42', $standard)->identifierAt());
116 self::assertNull(SqlTokenStream::tokenize("'value'", $standard)->identifierAt());
117 self::assertNull(SqlTokenStream::tokenize('[unfinished', $standard)->identifierAt());
118 self::assertNull(SqlTokenStream::tokenize('""', $allCapabilities)->identifierAt());
119 self::assertNull(SqlTokenStream::tokenize('``', $allCapabilities)->identifierAt());
120 }
121
122 public function testIdentifierAtUnescapesBracketAndAdvancesPastItsClosingToken(): void
123 {
124 self::assertSame(
125 ['name' => 'a]b', 'next' => 1],
126 SqlTokenStream::tokenize('[a]]b] tail', FakeSqlLexerProfiles::allCapabilities())->identifierAt(),
127 );
128 }
129
130 public function testSplitsOnlyTopLevelStatementTerminators(): void
131 {
132 $sql = 'SELECT \';\' AS value; SELECT $$a;b$$; /* ; */ SELECT (3; 4); SELECT [5; 6];;';
133 $profile = FakeSqlLexerProfiles::allCapabilities();
134
135 self::assertSame(
136 ["SELECT ';' AS value", 'SELECT $$a;b$$', '/* ; */ SELECT (3; 4)', 'SELECT [5; 6]'],
137 SqlTokenStream::tokenize($sql, $profile)->splitStatements(),
138 );
139 }
140
141 public function testReturnsNoStatementsForEmptyInputAndTerminators(): void
142 {
143 self::assertSame([], SqlTokenStream::tokenize(' ; ; ', FakeSqlLexerProfiles::standard())->splitStatements());
144 }
145
146 public function testUsesProfileSuppliedNestingAndPunctuation(): void
147 {
148 $profile = FakeSqlLexerProfiles::custom(
149 nestingPair: ['{', '}'],
150 statementDelimiter: '!',
151 listDelimiter: '|',
152 );
153 $nested = SqlTokenStream::tokenize('outer{inner{value}}', $profile);
154 $tokens = $nested->significantTokens();
155
156 self::assertSame('}', $nested->matchingClosingNestingToken($tokens[1])?->text);
157 self::assertSame(
158 ['alpha{nested!value}', 'beta'],
159 SqlTokenStream::tokenize('alpha{nested!value}!beta', $profile)->splitStatements(),
160 );
161 self::assertSame(
162 ['first', 'nested{second|third}', 'last'],
163 SqlTokenStream::tokenize('first|nested{second|third}|last', $profile)->splitTopLevel(),
164 );
165 }
166
167 public function testClauseIgnoresNestedKeywords(): void
168 {
169 $sql = "UPDATE users SET label = TRIM(BOTH 'x' FROM label), amount = CAST(raw AS DECIMAL(10,2)) WHERE id IN (SELECT id FROM source WHERE active) ORDER BY id";
170
171 $stream = SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard());
172
173 self::assertSame(
174 "label = TRIM(BOTH 'x' FROM label), amount = CAST(raw AS DECIMAL(10,2))",
175 $stream->topLevelClause(['SET'], [['FROM'], ['WHERE'], ['ORDER', 'BY'], ['LIMIT']]),
176 );
177 self::assertSame(
178 'id IN (SELECT id FROM source WHERE active)',
179 $stream->topLevelClause(['WHERE'], [['ORDER', 'BY'], ['LIMIT']]),
180 );
181 }
182
183 public function testClauseSupportsMultiWordBoundariesAndUsesEarliestEnd(): void
184 {
185 $sql = 'SELECT id FROM users ORDER BY created_at LIMIT 10';
186 $stream = SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard());
187
188 self::assertSame('created_at', $stream->topLevelClause(['ORDER', 'BY'], [['LIMIT'], ['RETURNING']]));
189 self::assertSame('id', $stream->topLevelClause(['SELECT'], [['LIMIT'], ['FROM']]));
190 self::assertNull($stream->topLevelClause(['GROUP', 'BY'], [['LIMIT']]));
191 }
192
193 public function testClauseRejectsSequencesSplitAcrossNestingLevels(): void
194 {
195 $sql = 'SELECT ORDER (BY hidden) BY visible FROM source';
196
197 self::assertNull(SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard())->topLevelClause(['ORDER', 'BY'], [['FROM']]));
198 }
199
200 public function testClauseCanStartAtLastAvailableToken(): void
201 {
202 self::assertSame('', SqlTokenStream::tokenize('SET', FakeSqlLexerProfiles::standard())->topLevelClause(['SET']));
203 }
204
205 public function testClauseAfterAnchorSkipsEarlierMatchingKeyword(): void
206 {
207 $sql = 'INSERT INTO items SELECT * FROM source WHERE active ON CONFLICT (id) DO UPDATE SET score = excluded.score WHERE items.score >= 80 RETURNING id';
208
209 self::assertSame(
210 'items.score >= 80',
211 SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard())->topLevelClauseAfter(
212 ['DO', 'UPDATE', 'SET'],
213 ['WHERE'],
214 [['RETURNING']],
215 ),
216 );
217 }
218
219 public function testClauseAfterAnchorCanBeginAtFirstToken(): void
220 {
221 self::assertSame(
222 'earlier',
223 SqlTokenStream::tokenize('DO UPDATE SET WHERE earlier WHERE later', FakeSqlLexerProfiles::standard())->topLevelClauseAfter(
224 ['DO', 'UPDATE', 'SET'],
225 ['WHERE'],
226 [['WHERE']],
227 ),
228 );
229 }
230
231 public function testSplitsCommasOutsideParenthesesArraysAndStrings(): void
232 {
233 $stream = SqlTokenStream::tokenize("ARRAY[1,2], COALESCE(a, b), 'x,y', plain", FakeSqlLexerProfiles::standard());
234
235 self::assertSame(
236 ['ARRAY[1,2]', 'COALESCE(a, b)', "'x,y'", 'plain'],
237 $stream->splitTopLevel(),
238 );
239 }
240
241 public function testSplitsWithCustomTopLevelDelimiterAndPreservesEmptyParts(): void
242 {
243 $stream = SqlTokenStream::tokenize(' first ; (nested ; value) ; ; last ', FakeSqlLexerProfiles::standard());
244
245 self::assertSame(
246 ['first', '(nested ; value)', '', 'last'],
247 $stream->splitTopLevel(';'),
248 );
249 }
250
251 public function testSplitTopLevelHandlesEmptyAndTrailingDelimiterInput(): void
252 {
253 self::assertSame([], SqlTokenStream::tokenize('', FakeSqlLexerProfiles::standard())->splitTopLevel());
254 self::assertSame(['value', ''], SqlTokenStream::tokenize('value,', FakeSqlLexerProfiles::standard())->splitTopLevel());
255 self::assertSame(['value'], SqlTokenStream::tokenize('value', FakeSqlLexerProfiles::standard())->splitTopLevel());
256 }
257
258 public function testTokenizesWhitespaceAndNestedCommentsLosslessly(): void
259 {
260 $sql = " \t-- note\n/* outer /* inner */ end */x";
261 $tokens = SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard())->tokens();
262
263 self::assertSame(
264 [
265 [SqlTokenKind::Whitespace, " \t", 0],
266 [SqlTokenKind::Comment, '-- note', 2],
267 [SqlTokenKind::Whitespace, "\n", 9],
268 [SqlTokenKind::Comment, '/* outer /* inner */ end */', 10],
269 [SqlTokenKind::Word, 'x', 37],
270 ],
271 array_map(
272 static fn (SqlToken $token): array => [$token->kind, $token->text, $token->offset],
273 $tokens,
274 ),
275 );
276 self::assertSame($sql, implode('', array_map(static fn (SqlToken $token): string => $token->text, $tokens)));
277 self::assertSame(
278 ['x'],
279 array_map(static fn (SqlToken $token): string => $token->text, SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard())->significantTokens()),
280 );
281 }
282
283 public function testTokenizesUnterminatedCommentsLosslessly(): void
284 {
285 self::assertSame(
286 [[SqlTokenKind::Comment, '/* outer /* inner */']],
287 array_map(
288 static fn (SqlToken $token): array => [$token->kind, $token->text],
289 SqlTokenStream::tokenize('/* outer /* inner */', FakeSqlLexerProfiles::standard())->tokens(),
290 ),
291 );
292 }
293
294 public function testCommentDelimitersDoNotOverlapOrConsumeFollowingTokens(): void
295 {
296 self::assertSame(
297 [
298 [SqlTokenKind::Comment, '/**/'],
299 [SqlTokenKind::Word, 'next'],
300 [SqlTokenKind::Comment, '/*/unterminated'],
301 ],
302 array_map(
303 static fn (SqlToken $token): array => [$token->kind, $token->text],
304 SqlTokenStream::tokenize('/**/next/*/unterminated', FakeSqlLexerProfiles::standard())->tokens(),
305 ),
306 );
307 self::assertSame(
308 [
309 [SqlTokenKind::Word, 'prefix'],
310 [SqlTokenKind::Whitespace, ' '],
311 [SqlTokenKind::Comment, '--'],
312 [SqlTokenKind::Whitespace, "\n"],
313 [SqlTokenKind::Word, 'next'],
314 ],
315 array_map(
316 static fn (SqlToken $token): array => [$token->kind, $token->text],
317 SqlTokenStream::tokenize("prefix --\nnext", FakeSqlLexerProfiles::standard())->tokens(),
318 ),
319 );
320 }
321
322 public function testAdditionalLineCommentsRemainProfileSpecific(): void
323 {
324 $sql = "# SELECT hidden\nDELETE FROM users WHERE payload #>> '$.name' = 'Alice'";
325 $profile = FakeSqlLexerProfiles::allCapabilities();
326
327 self::assertSame(
328 ['DELETE', 'FROM', 'users', 'WHERE', 'payload'],
329 array_map(
330 static fn (SqlToken $token): string => $token->text,
331 SqlTokenStream::tokenize($sql, $profile)->significantTokens(),
332 ),
333 );
334 self::assertSame(
335 ['#', 'SELECT', 'hidden', 'DELETE', 'FROM', 'users', 'WHERE', 'payload', '#', '>', '>', "'$.name'", '=', "'Alice'"],
336 array_map(
337 static fn (SqlToken $token): string => $token->text,
338 SqlTokenStream::tokenize($sql, FakeSqlLexerProfiles::standard())->significantTokens(),
339 ),
340 );
341 }
342
343 public function testNestedCommentDelimitersAdvanceAsPairs(): void
344 {
345 self::assertSame(
346 [
347 [SqlTokenKind::Comment, '/*/**/*/'],
348 [SqlTokenKind::Word, 'tail'],
349 [SqlTokenKind::Comment, '/*/*/x*/tail'],
350 ],
351 array_map(
352 static fn (SqlToken $token): array => [$token->kind, $token->text],
353 SqlTokenStream::tokenize('/*/**/*/tail/*/*/x*/tail', FakeSqlLexerProfiles::standard())->tokens(),
354 ),
355 );
356 }
357
358 public function testAsteriskWithoutOpeningSlashIsNotAComment(): void
359 {
360 self::assertSame(
361 [
362 [SqlTokenKind::Word, 'value'],
363 [SqlTokenKind::Symbol, '*'],
364 [SqlTokenKind::Symbol, '*'],
365 [SqlTokenKind::Word, 'not_comment'],
366 ],
367 array_map(
368 static fn (SqlToken $token): array => [$token->kind, $token->text],
369 SqlTokenStream::tokenize('value**not_comment', FakeSqlLexerProfiles::standard())->tokens(),
370 ),
371 );
372 }
373
374 public function testTokenizesQuotedValuesAndIdentifiers(): void
375 {
376 $sql = "'it''s' 'a\\\\\'b' \"a\"\"b\" `a``b` 'open";
377 $profile = FakeSqlLexerProfiles::allCapabilities();
378
379 self::assertSame(
380 [
381 [SqlTokenKind::String, "'it''s'"],
382 [SqlTokenKind::Whitespace, ' '],
383 [SqlTokenKind::String, "'a\\\\\'b'"],
384 [SqlTokenKind::Whitespace, ' '],
385 [SqlTokenKind::QuotedIdentifier, '"a""b"'],
386 [SqlTokenKind::Whitespace, ' '],
387 [SqlTokenKind::QuotedIdentifier, '`a``b`'],
388 [SqlTokenKind::Whitespace, ' '],
389 [SqlTokenKind::String, "'open"],
390 ],
391 array_map(
392 static fn (SqlToken $token): array => [$token->kind, $token->text],
393 SqlTokenStream::tokenize($sql, $profile)->tokens(),
394 ),
395 );
396 }
397
398 public function testIdentifierQuotesDoNotUseStringBackslashEscapes(): void
399 {
400 self::assertSame(
401 [
402 [SqlTokenKind::QuotedIdentifier, '"a\\"'],
403 [SqlTokenKind::Whitespace, ' '],
404 [SqlTokenKind::Word, 'tail'],
405 ],
406 array_map(
407 static fn (SqlToken $token): array => [$token->kind, $token->text],
408 SqlTokenStream::tokenize('"a\\" tail', FakeSqlLexerProfiles::allCapabilities())->tokens(),
409 ),
410 );
411 }
412
413 public function testQuotedEscapeAtEndRemainsInsideUnterminatedValue(): void
414 {
415 $profile = FakeSqlLexerProfiles::allCapabilities();
416
417 self::assertSame(
418 [[SqlTokenKind::String, "'value\x5c"]],
419 array_map(
420 static fn (SqlToken $token): array => [$token->kind, $token->text],
421 SqlTokenStream::tokenize("'value\x5c", $profile)->tokens(),
422 ),
423 );
424 }
425
426 public function testQuotedEscapeControlsWhereTheValueEnds(): void
427 {
428 $profile = FakeSqlLexerProfiles::allCapabilities();
429
430 self::assertSame(
431 [
432 [SqlTokenKind::String, "'''\x5c'x'"],
433 [SqlTokenKind::Whitespace, ' '],
434 [SqlTokenKind::String, "'\x5cx'"],
435 [SqlTokenKind::Whitespace, ' '],
436 [SqlTokenKind::Word, 'tail'],
437 ],
438 array_map(
439 static fn (SqlToken $token): array => [$token->kind, $token->text],
440 SqlTokenStream::tokenize("'''\x5c'x' '\x5cx' tail", $profile)->tokens(),
441 ),
442 );
443 }
444
445 public function testTokenizesDollarStringsAndParameters(): void
446 {
447 $sql = '$$body$$ $tag$body$tag$ $1 $123 ? :name :: : $word';
448 $profile = FakeSqlLexerProfiles::allCapabilities();
449
450 self::assertSame(
451 [
452 [SqlTokenKind::String, '$$body$$'],
453 [SqlTokenKind::String, '$tag$body$tag$'],
454 [SqlTokenKind::Parameter, '$1'],
455 [SqlTokenKind::Parameter, '$123'],
456 [SqlTokenKind::Parameter, '?'],
457 [SqlTokenKind::Parameter, ':name'],
458 [SqlTokenKind::Symbol, ':'],
459 [SqlTokenKind::Symbol, ':'],
460 [SqlTokenKind::Symbol, ':'],
461 [SqlTokenKind::Symbol, '$'],
462 [SqlTokenKind::Word, 'word'],
463 ],
464 array_map(
465 static fn (SqlToken $token): array => [$token->kind, $token->text],
466 SqlTokenStream::tokenize($sql, $profile)->significantTokens(),
467 ),
468 );
469 }
470
471 public function testParametersStopBeforeFollowingWordAndAtEndOfInput(): void
472 {
473 $profile = FakeSqlLexerProfiles::allCapabilities();
474
475 self::assertSame(
476 [
477 [SqlTokenKind::Parameter, '$1'],
478 [SqlTokenKind::Word, 'x'],
479 [SqlTokenKind::Parameter, ':end'],
480 ],
481 array_map(
482 static fn (SqlToken $token): array => [$token->kind, $token->text],
483 SqlTokenStream::tokenize('$1x :end', $profile)->significantTokens(),
484 ),
485 );
486 }
487
488 public function testTokenizesNumberedQuestionMarkAndStructuredNamedParameters(): void
489 {
490 self::assertSame(
491 [
492 [SqlTokenKind::Parameter, '?123'],
493 [SqlTokenKind::Parameter, ':x::part(payload)'],
494 ],
495 array_map(
496 static fn (SqlToken $token): array => [$token->kind, $token->text],
497 SqlTokenStream::tokenize('?123 :x::part(payload)', FakeSqlLexerProfiles::allCapabilities())->significantTokens(),
498 ),
499 );
500 }
501
502 public function testQuestionMarkProfileCanRejectNumericSuffixes(): void
503 {
504 $profile = FakeSqlLexerProfiles::custom(
505 positionalParameterPatterns: ['/^\?/'],
506 );
507
508 self::assertSame(
509 [
510 [SqlTokenKind::Parameter, '?'],
511 [SqlTokenKind::Number, '123'],
512 ],
513 array_map(
514 static fn (SqlToken $token): array => [$token->kind, $token->text],
515 SqlTokenStream::tokenize('?123', $profile)->significantTokens(),
516 ),
517 );
518 }
519
520 public function testTokenizesUnterminatedDollarString(): void
521 {
522 $profile = FakeSqlLexerProfiles::allCapabilities();
523
524 self::assertSame(
525 [[SqlTokenKind::String, '$tag$body']],
526 array_map(
527 static fn (SqlToken $token): array => [$token->kind, $token->text],
528 SqlTokenStream::tokenize('$tag$body', $profile)->tokens(),
529 ),
530 );
531 }
532
533 public function testDollarTagMustStartAtCurrentOffset(): void
534 {
535 $profile = FakeSqlLexerProfiles::allCapabilities();
536
537 self::assertSame(
538 [
539 [SqlTokenKind::Symbol, '$'],
540 [SqlTokenKind::Word, 'bad'],
541 [SqlTokenKind::Symbol, '-'],
542 [SqlTokenKind::String, '$tag$body$tag$'],
543 ],
544 array_map(
545 static fn (SqlToken $token): array => [$token->kind, $token->text],
546 SqlTokenStream::tokenize('$bad-$tag$body$tag$', $profile)->tokens(),
547 ),
548 );
549 }
550
551 public function testTokenizesWordsAndConfiguredNumberForms(): void
552 {
553 $sql = "_word2\$tail café9 \x80 0 9_000 0xCA_FE 0Xb 0y 1.25 2e3 3E+4 4e-5 5. 6eX";
554 $profile = FakeSqlLexerProfiles::allCapabilities();
555
556 self::assertSame(
557 [
558 [SqlTokenKind::Word, '_word2$tail'],
559 [SqlTokenKind::Word, 'café9'],
560 [SqlTokenKind::Word, "\x80"],
561 [SqlTokenKind::Number, '0'],
562 [SqlTokenKind::Number, '9_000'],
563 [SqlTokenKind::Number, '0xCA_FE'],
564 [SqlTokenKind::Number, '0Xb'],
565 [SqlTokenKind::Number, '0'],
566 [SqlTokenKind::Word, 'y'],
567 [SqlTokenKind::Number, '1.25'],
568 [SqlTokenKind::Number, '2e3'],
569 [SqlTokenKind::Number, '3E+4'],
570 [SqlTokenKind::Number, '4e-5'],
571 [SqlTokenKind::Number, '5.'],
572 [SqlTokenKind::Number, '6'],
573 [SqlTokenKind::Word, 'eX'],
574 ],
575 array_map(
576 static fn (SqlToken $token): array => [$token->kind, $token->text],
577 SqlTokenStream::tokenize($sql, $profile)->significantTokens(),
578 ),
579 );
580 }
581
582 public function testTracksParenthesisAndBracketDepthWithoutUnderflow(): void
583 {
584 self::assertSame(
585 [
586 ['(', 0, 0],
587 ['[', 1, 0],
588 ['x', 1, 1],
589 [']', 1, 0],
590 [',', 1, 0],
591 ['(', 1, 0],
592 ['y', 2, 0],
593 [')', 1, 0],
594 [')', 0, 0],
595 ['[', 0, 0],
596 ['[', 0, 1],
597 ['z', 0, 2],
598 [']', 0, 1],
599 [']', 0, 0],
600 [')', 0, 0],
601 [']', 0, 0],
602 ['(', 0, 0],
603 ],
604 array_map(
605 static fn (SqlToken $token): array => [$token->text, $token->depth, $token->bracketDepth],
606 SqlTokenStream::tokenize('([x],(y))[[z]])](', FakeSqlLexerProfiles::standard())->significantTokens(),
607 ),
608 );
609 }
610
611 public function testFindsFirstTopLevelKeywordAfterLeadingComment(): void
612 {
613 self::assertSame(
614 'WITH',
615 SqlTokenStream::tokenize('/* SELECT */ WITH data AS (SELECT 1) SELECT * FROM data', FakeSqlLexerProfiles::standard())->firstTopLevelKeyword(),
616 );
617 }
618
619 public function testFirstTopLevelKeywordIgnoresNestedWordsAndCanBeAbsent(): void
620 {
621 self::assertSame('UPDATE', SqlTokenStream::tokenize('(SELECT hidden) [DELETE hidden] update users', FakeSqlLexerProfiles::standard())->firstTopLevelKeyword());
622 self::assertNull(SqlTokenStream::tokenize("123 + 'SELECT'", FakeSqlLexerProfiles::standard())->firstTopLevelKeyword());
623 }
624
625 public function testKeepsArithmeticOperatorsOutsideNumberTokens(): void
626 {
627 $tokens = SqlTokenStream::tokenize('SELECT 1+2, 3.5e-2', FakeSqlLexerProfiles::standard())->significantTokens();
628
629 self::assertSame(
630 ['SELECT', '1', '+', '2', ',', '3.5e-2'],
631 array_map(static fn (SqlToken $token): string => $token->text, $tokens),
632 );
633 }
634
635 public function testTokensKeepsEveryLexemeIncludingTheSpaceBetweenThem(): void
636 {
637 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
638
639 self::assertGreaterThan(count($stream->significantTokens()), count($stream->tokens()));
640 }
641
642 public function testSignificantTokensLeavesOutWhitespaceAndComments(): void
643 {
644 $stream = SqlTokenStream::tokenize('SELECT /* c */ 1', FakeSqlLexerProfiles::standard());
645
646 $texts = array_map(static fn (SqlToken $t): string => $t->text, $stream->significantTokens());
647
648 self::assertSame(['SELECT', '1'], $texts);
649 }
650
651 public function testSignificantTokenBeforeAnswersTheLexemeThatCameFirst(): void
652 {
653 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
654 $tokens = $stream->significantTokens();
655
656 self::assertSame($tokens[0], $stream->significantTokenBefore($tokens[1]));
657 }
658
659 public function testSignificantTokenBeforeIsNothingForTheFirstLexeme(): void
660 {
661 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
662 $tokens = $stream->significantTokens();
663
664 self::assertNull($stream->significantTokenBefore($tokens[0]));
665 }
666
667 public function testSignificantTokenAfterAnswersTheLexemeThatComesNext(): void
668 {
669 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
670 $tokens = $stream->significantTokens();
671
672 self::assertSame($tokens[1], $stream->significantTokenAfter($tokens[0]));
673 }
674
675 public function testSignificantTokenAfterIsNothingForTheLastLexeme(): void
676 {
677 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
678 $tokens = $stream->significantTokens();
679
680 self::assertNull($stream->significantTokenAfter($tokens[count($tokens) - 1]));
681 }
682
683 public function testMatchingClosingNestingTokenAnswersTheParenthesisThatClosesOne(): void
684 {
685 $stream = SqlTokenStream::tokenize('SELECT (1 + (2))', FakeSqlLexerProfiles::standard());
686 $tokens = $stream->significantTokens();
687
688 $closing = $stream->matchingClosingNestingToken($tokens[1]);
689
690 self::assertSame([')', 15], [$closing?->text, $closing?->offset]);
691 }
692
693 public function testMatchingClosingNestingTokenIsNothingForALexemeThatOpensNothing(): void
694 {
695 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
696 $tokens = $stream->significantTokens();
697
698 self::assertNull($stream->matchingClosingNestingToken($tokens[0]));
699 }
700
701 public function testSplitStatementsReadsABatchAsTheStatementsItIsWrittenAs(): void
702 {
703 $stream = SqlTokenStream::tokenize('SELECT 1; SELECT 2', FakeSqlLexerProfiles::standard());
704
705 self::assertSame(['SELECT 1', 'SELECT 2'], array_map(trim(...), $stream->splitStatements()));
706 }
707
708 public function testSplitStatementsLeavesASemicolonInsideAStringAlone(): void
709 {
710 $stream = SqlTokenStream::tokenize("SELECT ';'", FakeSqlLexerProfiles::standard());
711
712 self::assertCount(1, $stream->splitStatements());
713 }
714
715 public function testTopLevelClauseAnswersTheTextBetweenTheKeywordsThatBoundIt(): void
716 {
717 $stream = SqlTokenStream::tokenize(
718 'SELECT * FROM users WHERE id = 1 ORDER BY id',
719 FakeSqlLexerProfiles::standard(),
720 );
721
722 self::assertSame('id = 1', trim((string) $stream->topLevelClause(['WHERE'], [['ORDER']])));
723 }
724
725 public function testTopLevelClauseIsNothingWhereTheKeywordIsNotWritten(): void
726 {
727 $stream = SqlTokenStream::tokenize('SELECT 1', FakeSqlLexerProfiles::standard());
728
729 self::assertNull($stream->topLevelClause(['WHERE']));
730 }
731
732 public function testTopLevelClauseAfterLooksOnlyPastTheAnchorItWasGiven(): void
733 {
734 $stream = SqlTokenStream::tokenize(
735 'UPDATE users SET name = \'a\' WHERE id = 1',
736 FakeSqlLexerProfiles::standard(),
737 );
738
739 self::assertSame('id = 1', trim((string) $stream->topLevelClauseAfter(['SET'], ['WHERE'], [])));
740 }
741
742 public function testTopLevelClauseAfterIsNothingWhereTheAnchorIsNotWritten(): void
743 {
744 $stream = SqlTokenStream::tokenize('SELECT 1 WHERE 1', FakeSqlLexerProfiles::standard());
745
746 self::assertNull($stream->topLevelClauseAfter(['SET'], ['WHERE'], []));
747 }
748}
749