packages/sql-faker/tests/Unit/MySql/Generation/Tokenization/MySqlTokenizerTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\SqlFaker\MySql\Generation\Tokenization;
6
7use PHPUnit\Framework\Attributes\CoversClass;
8use PHPUnit\Framework\Attributes\DataProvider;
9use PHPUnit\Framework\Attributes\UsesClass;
10use PHPUnit\Framework\TestCase;
11use SqlFaker\Generation\Exception\LexicalException;
12use SqlFaker\MySql\Generation\Tokenization\MySqlTokenizer;
13
14#[CoversClass(MySqlTokenizer::class)]
15#[UsesClass(LexicalException::class)]
16final class MySqlTokenizerTest extends TestCase
17{
18 /**
19 * @param list<string> $expected
20 */
21 #[DataProvider('providerSql')]
22 public function testTokenizeReadsTextAsTheServerWould(string $sql, array $expected): void
23 {
24 $tokenizer = new MySqlTokenizer(
25 ['SELECT' => 'SELECT_SYM', 'WITH' => 'WITH', 'ROLLUP' => 'ROLLUP_SYM', '<=>' => 'EQUAL_SYM'],
26 ['COUNT' => 'COUNT_SYM'],
27 true,
28 );
29
30 self::assertSame($expected, $tokenizer->tokenize($sql));
31 }
32
33 /**
34 * @return iterable<string, array{string, list<string>}>
35 */
36 public static function providerSql(): iterable
37 {
38 yield 'nothing' => ['', []];
39 yield 'keyword' => ['SELECT', ['SELECT_SYM']];
40 yield 'keyword regardless of case' => ['select', ['SELECT_SYM']];
41 yield 'identifier' => ['users', ['IDENT']];
42 yield 'quoted identifier' => ['`users`', ['IDENT_QUOTED']];
43 yield 'string literal' => ["'text'", ['TEXT_STRING']];
44 yield 'parameter marker' => ['?', ['PARAM_MARKER']];
45 yield 'charset introducer' => ['_utf8mb4', ['UNDERSCORE_CHARSET']];
46 yield 'ASCII charset introducer' => ['_ascii', ['UNDERSCORE_CHARSET']];
47 yield 'UTF8MB3 charset introducer' => ['_utf8mb3', ['UNDERSCORE_CHARSET']];
48 yield 'Latin1 charset introducer' => ['_LATIN1', ['UNDERSCORE_CHARSET']];
49 yield 'binary charset introducer' => ['_binary', ['UNDERSCORE_CHARSET']];
50 yield 'other underscore word' => ['_other', ['IDENT']];
51 yield 'hex in 0x form' => ['0xFF', ['HEX_NUM']];
52 yield 'hex in quoted form' => ["X'FF'", ['HEX_NUM']];
53 yield 'binary in 0b form' => ['0b01', ['BIN_NUM']];
54 yield 'binary in quoted form' => ["B'01'", ['BIN_NUM']];
55 yield 'national string' => ["N'text'", ['NCHAR_STRING']];
56 yield 'decimal' => ['1.5', ['DECIMAL_NUM']];
57 yield 'float' => ['1.5e3', ['FLOAT_NUM']];
58 yield 'operator' => ['<=>', ['EQUAL_SYM']];
59 yield 'punctuation' => ['(', ['(']];
60 yield 'function before a parenthesis' => ['COUNT(', ['COUNT_SYM', '(']];
61 yield 'function name used as an identifier' => ['COUNT ', ['IDENT']];
62 yield 'user variable' => ['@name', ['@', 'LEX_HOSTNAME']];
63 yield 'with rollup is one token' => ['WITH ROLLUP', ['WITH_ROLLUP_SYM']];
64 yield 'dollar quoted string' => ['$$text$$', ['DOLLAR_QUOTED_STRING_SYM']];
65 yield 'whitespace separates nothing' => [' SELECT users ', ['SELECT_SYM', 'IDENT']];
66 yield 'block comment is trivia' => ['SELECT /* note */ users', ['SELECT_SYM', 'IDENT']];
67 yield 'hash comment is trivia' => ["SELECT # note\nusers", ['SELECT_SYM', 'IDENT']];
68 yield 'dash comment is trivia' => ["SELECT -- note\nusers", ['SELECT_SYM', 'IDENT']];
69 }
70
71 public function testTokenizeRefusesADollarQuotedStringWhenTheBuildDoesNot(): void
72 {
73 $tokenizer = new MySqlTokenizer([], [], false);
74
75 $this->expectException(LexicalException::class);
76 $this->expectExceptionMessage('Unsupported MySQL lexical input at offset 0:');
77
78 $tokenizer->tokenize('$$text$$');
79 }
80
81 public function testTokenizeReportsTextItCannotRead(): void
82 {
83 $tokenizer = new MySqlTokenizer([], [], true);
84
85 $this->expectException(LexicalException::class);
86 $this->expectExceptionMessage('Unsupported MySQL lexical input at offset 0:');
87
88 $tokenizer->tokenize('\\');
89 }
90
91 public function testTokenizeReportsAnUnterminatedDollarQuotedString(): void
92 {
93 $tokenizer = new MySqlTokenizer([], [], true);
94
95 $this->expectException(LexicalException::class);
96 $this->expectExceptionMessage('Unterminated MySQL dollar-quoted string.');
97
98 $tokenizer->tokenize('$$text');
99 }
100
101 public function testQuotedTokenAtLeavesAnythingThatOpensNoQuote(): void
102 {
103 $offset = 0;
104
105 self::assertNull((new MySqlTokenizer([], [], true))->quotedTokenAt('users', $offset));
106 self::assertSame(0, $offset);
107 }
108
109 public function testMarkerTokenAtReadsAHostnameOnlyAfterASigil(): void
110 {
111 $offset = 0;
112
113 self::assertSame('LEX_HOSTNAME', (new MySqlTokenizer([], [], true))->markerTokenAt('localhost', $offset, ['@']));
114 self::assertSame(9, $offset);
115 }
116
117 public function testMarkerTokenAtLeavesTheSameCharactersAloneWithoutASigil(): void
118 {
119 $offset = 0;
120
121 self::assertNull((new MySqlTokenizer([], [], true))->markerTokenAt('localhost', $offset, []));
122 self::assertSame(0, $offset);
123 }
124
125 public function testNumericTokenAtLeavesTextThatIsNoNumber(): void
126 {
127 $offset = 0;
128
129 self::assertNull((new MySqlTokenizer([], [], true))->numericTokenAt('users', $offset));
130 }
131
132 public function testWordTokenAtLeavesTextThatIsNoWord(): void
133 {
134 $offset = 0;
135
136 self::assertNull((new MySqlTokenizer([], [], true))->wordTokenAt('123', $offset));
137 }
138
139 public function testOperatorTokenAtLeavesTextThatIsNoOperator(): void
140 {
141 $offset = 0;
142
143 self::assertNull((new MySqlTokenizer([], [], true))->operatorTokenAt('users', $offset));
144 }
145
146 public function testTokenAtReadsTheFirstTokenAndAdvancesPastIt(): void
147 {
148 $tokenizer = new MySqlTokenizer(['SELECT' => 'SELECT_SYM'], [], true);
149 $offset = 0;
150
151 self::assertSame('SELECT_SYM', $tokenizer->tokenAt('SELECT users', $offset, []));
152 self::assertSame(6, $offset);
153 }
154
155 public function testSkipTriviaReportsWhenNothingWasSkipped(): void
156 {
157 $offset = 0;
158
159 self::assertFalse((new MySqlTokenizer([], [], true))->skipTrivia('SELECT', $offset));
160 self::assertSame(0, $offset);
161 }
162
163 public function testSkipTriviaReportsAnUnterminatedBlockComment(): void
164 {
165 $offset = 0;
166
167 $this->expectException(LexicalException::class);
168 $this->expectExceptionMessage('Unterminated MySQL block comment.');
169
170 (new MySqlTokenizer([], [], true))->skipTrivia('/* never closed', $offset);
171 }
172
173 public function testSkipQuotedTreatsADoubledQuoteAsAnEscape(): void
174 {
175 $offset = 0;
176
177 (new MySqlTokenizer([], [], true))->skipQuoted("'a''b' rest", $offset, "'");
178
179 self::assertSame(6, $offset);
180 }
181
182 public function testSkipQuotedReportsARunThatNeverCloses(): void
183 {
184 $offset = 0;
185
186 $this->expectException(LexicalException::class);
187 $this->expectExceptionMessage('Unterminated MySQL quoted token');
188
189 (new MySqlTokenizer([], [], true))->skipQuoted("'abc", $offset, "'");
190 }
191
192 #[DataProvider('providerInteger')]
193 public function testIntegerTokenWidensByMagnitude(string $digits, string $expected): void
194 {
195 self::assertSame($expected, (new MySqlTokenizer([], [], true))->integerToken($digits));
196 }
197
198 /**
199 * @return iterable<string, array{string, string}>
200 */
201 public static function providerInteger(): iterable
202 {
203 yield 'zero' => ['0', 'NUM'];
204 yield 'leading zeroes do not widen it' => ['0000000000000000001', 'NUM'];
205 yield 'the largest NUM' => ['2147483647', 'NUM'];
206 yield 'one past the largest NUM' => ['2147483648', 'LONG_NUM'];
207 yield 'the largest LONG_NUM' => ['9223372036854775807', 'LONG_NUM'];
208 yield 'one past the largest LONG_NUM' => ['9223372036854775808', 'ULONGLONG_NUM'];
209 yield 'the largest ULONGLONG_NUM' => ['18446744073709551615', 'ULONGLONG_NUM'];
210 yield 'first overflowing unsigned integer' => ['18446744073709551616', 'DECIMAL_NUM'];
211 yield 'wider overflow' => [str_repeat('9', 100), 'DECIMAL_NUM'];
212 }
213
214 public function testOperatorAtPrefersTheLongestOperator(): void
215 {
216 $tokenizer = new MySqlTokenizer([], [], true);
217
218 self::assertSame('<=>', $tokenizer->operatorAt('<=>', 0));
219 self::assertSame('<=', $tokenizer->operatorAt('<=x', 0));
220 self::assertSame('<', $tokenizer->operatorAt('<x', 0));
221 }
222
223 public function testOperatorAtReportsNothingForAWord(): void
224 {
225 self::assertNull((new MySqlTokenizer([], [], true))->operatorAt('users', 0));
226 }
227
228 public function testIsPunctuationAcceptsASingleGrammarCharacter(): void
229 {
230 $tokenizer = new MySqlTokenizer([], [], true);
231
232 self::assertTrue($tokenizer->isPunctuation('('));
233 self::assertFalse($tokenizer->isPunctuation('a'));
234 self::assertFalse($tokenizer->isPunctuation('<='));
235 }
236
237 public function testMergedLeavesTokensThatDoNotPairUp(): void
238 {
239 self::assertSame(
240 ['SELECT_SYM', 'IDENT'],
241 (new MySqlTokenizer([], [], true))->merged(['SELECT_SYM', 'IDENT']),
242 );
243 }
244
245 public function testMergedJoinsWithRollup(): void
246 {
247 self::assertSame(['WITH_ROLLUP_SYM'], (new MySqlTokenizer([], [], true))->merged(['WITH', 'ROLLUP_SYM']));
248 }
249}
250