packages/sql-faker/tests/Unit/PostgreSql/Generation/Tokenization/PgTokenizerTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\SqlFaker\PostgreSql\Generation\Tokenization;
6
7use PHPUnit\Framework\Attributes\CoversClass;
8use PHPUnit\Framework\Attributes\DataProvider;
9use PHPUnit\Framework\Attributes\UsesClass;
10use PHPUnit\Framework\TestCase;
11use SqlFaker\Generation\Exception\LexicalException;
12use SqlFaker\PostgreSql\Generation\Lookahead\PgLookahead;
13use SqlFaker\PostgreSql\Generation\Tokenization\PgTokenizer;
14
15#[CoversClass(PgTokenizer::class)]
16#[UsesClass(LexicalException::class)]
17#[UsesClass(PgLookahead::class)]
18final class PgTokenizerTest extends TestCase
19{
20 /**
21 * @param list<string> $expected
22 */
23 #[DataProvider('providerSql')]
24 public function testTokenizeReadsTextAsTheServerWould(string $sql, array $expected): void
25 {
26 $tokenizer = new PgTokenizer(['SELECT' => 'SELECT', 'NOT' => 'NOT'], new PgLookahead([]));
27
28 self::assertSame($expected, $tokenizer->tokenize($sql));
29 }
30
31 /**
32 * @return iterable<string, array{string, list<string>}>
33 */
34 public static function providerSql(): iterable
35 {
36 yield 'nothing' => ['', []];
37 yield 'keyword' => ['SELECT', ['SELECT']];
38 yield 'keyword regardless of case' => ['select', ['SELECT']];
39 yield 'identifier' => ['users', ['IDENT']];
40 yield 'quoted identifier' => ['"users"', ['IDENT']];
41 yield 'unicode quoted identifier' => ['U&"users"', ['IDENT']];
42 yield 'string literal' => ["'text'", ['SCONST']];
43 yield 'unicode string literal' => ["U&'text'", ['SCONST']];
44 yield 'escaped string literal' => ["E'text'", ['SCONST']];
45 yield 'bit string literal' => ["B'01'", ['BCONST']];
46 yield 'hex string literal' => ["X'FF'", ['XCONST']];
47 yield 'dollar quoted string' => ['$$text$$', ['SCONST']];
48 yield 'tagged dollar quoted string' => ['$tag$text$tag$', ['SCONST']];
49 yield 'positional parameter' => ['$1', ['PARAM']];
50 yield 'integer' => ['42', ['ICONST']];
51 yield 'integer digit separators' => ['1_0', ['ICONST']];
52 yield 'largest integer' => ['2_147_483_647', ['ICONST']];
53 yield 'first overflow integer' => ['2_147_483_648', ['FCONST']];
54 yield 'leading zeroes retain magnitude' => ['00000000000000000000001', ['ICONST']];
55 yield 'large overflow integer' => [str_repeat('9', 100), ['FCONST']];
56 yield 'float' => ['1.5', ['FCONST']];
57 yield 'exponent float' => ['1e3', ['FCONST']];
58 yield 'typecast' => ['::', ['TYPECAST']];
59 yield 'range' => ['..', ['DOT_DOT']];
60 yield 'assignment' => [':=', ['COLON_EQUALS']];
61 yield 'named argument' => ['=>', ['EQUALS_GREATER']];
62 yield 'inequality' => ['<>', ['NOT_EQUALS']];
63 yield 'user operator' => ['?|', ['Op']];
64 yield 'named argument prefix in a longer operator' => ['=>@', ['Op']];
65 yield 'inequality prefix in a longer operator' => ['<>*', ['Op']];
66 yield 'comparison prefix in a longer operator' => ['<=?', ['Op']];
67 yield 'SQL operator before a sign' => ['=-1', ['=', '-', 'ICONST']];
68 yield 'comparison before a sign' => ['<=+1', ['LESS_EQUALS', '+', 'ICONST']];
69 yield 'all signs are separate operators' => ['++1', ['+', '+', 'ICONST']];
70 yield 'non-SQL operator retains a trailing sign' => ['?-1', ['Op', 'ICONST']];
71 yield 'named argument stops before a comment' => ['=>/* note */1', ['EQUALS_GREATER', 'ICONST']];
72 yield 'punctuation' => ['(', ['(']];
73 yield 'line comment is trivia' => ["SELECT -- note\nusers", ['SELECT', 'IDENT']];
74 yield 'block comment is trivia' => ['SELECT /* note */ users', ['SELECT', 'IDENT']];
75 yield 'block comments nest' => ['SELECT /* a /* b */ c */ users', ['SELECT', 'IDENT']];
76 yield 'operator stops before a comment' => ["+-- note\nusers", ['+', 'IDENT']];
77 }
78
79 public function testTokenizeAppliesTheLookaheadSubstitution(): void
80 {
81 $lookahead = new PgLookahead(['NOT' => ['token' => 'NOT_LA', 'followed_by' => ['NULL_P']]]);
82 $tokenizer = new PgTokenizer(['NOT' => 'NOT', 'NULL' => 'NULL_P'], $lookahead);
83
84 self::assertSame(['NOT_LA', 'NULL_P'], $tokenizer->tokenize('NOT NULL'));
85 }
86
87 public function testTokenizeReportsTextItCannotRead(): void
88 {
89 $tokenizer = new PgTokenizer([], new PgLookahead([]));
90
91 $this->expectException(LexicalException::class);
92 $this->expectExceptionMessage('Unsupported PostgreSQL lexical input at offset 0:');
93
94 $tokenizer->tokenize('\\');
95 }
96
97 public function testTokenizeReportsAnUnterminatedDollarQuotedString(): void
98 {
99 $tokenizer = new PgTokenizer([], new PgLookahead([]));
100
101 $this->expectException(LexicalException::class);
102 $this->expectExceptionMessage('Unterminated PostgreSQL dollar-quoted string.');
103
104 $tokenizer->tokenize('$$text');
105 }
106
107 public function testQuotedTokenAtLeavesAnythingThatOpensNoQuote(): void
108 {
109 $offset = 0;
110
111 self::assertNull((new PgTokenizer([], new PgLookahead([])))->quotedTokenAt('users', $offset));
112 self::assertSame(0, $offset);
113 }
114
115 public function testDollarTokenAtLeavesADollarThatOpensNeither(): void
116 {
117 $offset = 0;
118
119 self::assertNull((new PgTokenizer([], new PgLookahead([])))->dollarTokenAt('users', $offset));
120 }
121
122 public function testNumericTokenAtLeavesTextThatIsNoNumber(): void
123 {
124 $offset = 0;
125
126 self::assertNull((new PgTokenizer([], new PgLookahead([])))->numericTokenAt('users', $offset));
127 }
128
129 public function testWordTokenAtLeavesTextThatIsNoWord(): void
130 {
131 $offset = 0;
132
133 self::assertNull((new PgTokenizer([], new PgLookahead([])))->wordTokenAt('42', $offset));
134 }
135
136 public function testOperatorTokenAtLeavesTextThatIsNoOperator(): void
137 {
138 $offset = 0;
139
140 self::assertNull((new PgTokenizer([], new PgLookahead([])))->operatorTokenAt('users', $offset));
141 }
142
143 public function testTokenAtReadsTheFirstTokenAndAdvancesPastIt(): void
144 {
145 $tokenizer = new PgTokenizer(['SELECT' => 'SELECT'], new PgLookahead([]));
146 $offset = 0;
147
148 self::assertSame('SELECT', $tokenizer->tokenAt('SELECT users', $offset));
149 self::assertSame(6, $offset);
150 }
151
152 public function testSkipTriviaReportsWhenNothingWasSkipped(): void
153 {
154 $offset = 0;
155
156 self::assertFalse((new PgTokenizer([], new PgLookahead([])))->skipTrivia('SELECT', $offset));
157 }
158
159 public function testSkipTriviaReportsAnUnterminatedBlockComment(): void
160 {
161 $offset = 0;
162
163 $this->expectException(LexicalException::class);
164 $this->expectExceptionMessage('Unterminated PostgreSQL block comment.');
165
166 (new PgTokenizer([], new PgLookahead([])))->skipTrivia('/* never closed', $offset);
167 }
168
169 public function testSkipQuotedTreatsADoubledQuoteAsAnEscape(): void
170 {
171 $offset = 0;
172
173 (new PgTokenizer([], new PgLookahead([])))->skipQuoted("'a''b' rest", $offset, "'");
174
175 self::assertSame(6, $offset);
176 }
177
178 public function testSkipQuotedReportsARunThatNeverCloses(): void
179 {
180 $offset = 0;
181
182 $this->expectException(LexicalException::class);
183 $this->expectExceptionMessage('Unterminated PostgreSQL quoted token');
184
185 (new PgTokenizer([], new PgLookahead([])))->skipQuoted("'abc", $offset, "'");
186 }
187
188 public function testOperatorAtTakesTheLongestRunOfOperatorCharacters(): void
189 {
190 self::assertSame(['+*', 'Op'], (new PgTokenizer([], new PgLookahead([])))->operatorAt('+*x', 0));
191 }
192
193 public function testOperatorAtReportsPunctuationAsItself(): void
194 {
195 self::assertSame(['(', '('], (new PgTokenizer([], new PgLookahead([])))->operatorAt('(x', 0));
196 }
197
198 public function testOperatorAtReportsNothingForAWord(): void
199 {
200 self::assertNull((new PgTokenizer([], new PgLookahead([])))->operatorAt('users', 0));
201 }
202
203 public function testFixedOperatorNamesTheOperatorsThatHaveTokens(): void
204 {
205 $tokenizer = new PgTokenizer([], new PgLookahead([]));
206
207 self::assertSame('TYPECAST', $tokenizer->fixedOperator('::'));
208 self::assertSame('NOT_EQUALS', $tokenizer->fixedOperator('!='));
209 self::assertNull($tokenizer->fixedOperator('?|'));
210 }
211
212 public function testIsPunctuationAcceptsASingleGrammarCharacter(): void
213 {
214 $tokenizer = new PgTokenizer([], new PgLookahead([]));
215
216 self::assertTrue($tokenizer->isPunctuation('('));
217 self::assertFalse($tokenizer->isPunctuation('a'));
218 self::assertFalse($tokenizer->isPunctuation('::'));
219 }
220}
221