packages/sql-faker/tests/Unit/Sqlite/Generation/LexicalGrammarTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\SqlFaker\Sqlite\Generation;
6
7use Closure;
8use Faker\Factory;
9use PHPUnit\Framework\Attributes\CoversClass;
10use PHPUnit\Framework\Attributes\DataProvider;
11use PHPUnit\Framework\Attributes\UsesClass;
12use PHPUnit\Framework\TestCase;
13use SqlFaker\Generation\Exception\LexicalException;
14use SqlFaker\Generation\Plan\GenerationPlan;
15use SqlFaker\Generation\Token\TerminalSequence;
16use SqlFaker\Generation\Value\RandomCharacters;
17use SqlFaker\Grammar\Resource\SqlVersion;
18use SqlFaker\Grammar\Resource\SqlVersionRegistry;
19use SqlFaker\Sqlite\Generation\LexicalGrammar;
20use SqlFaker\Sqlite\Generation\Tokenization\KeywordIndex;
21use SqlFaker\Sqlite\Generation\Tokenization\SqliteTokenizer;
22use SqlFaker\Sqlite\Generation\Value\LiteralGenerator;
23
24#[CoversClass(LexicalGrammar::class)]
25#[CoversClass(LiteralGenerator::class)]
26#[UsesClass(SqlVersion::class)]
27#[UsesClass(LexicalException::class)]
28#[UsesClass(KeywordIndex::class)]
29#[UsesClass(RandomCharacters::class)]
30#[UsesClass(SqlVersionRegistry::class)]
31#[UsesClass(SqliteTokenizer::class)]
32#[UsesClass(\SqlFaker\Sqlite\Generation\Tokenization\SqliteQuoting::class)]
33#[UsesClass(GenerationPlan::class)]
34#[UsesClass(\SqlFaker\Generation\Candidate\ChoiceLexemeGenerator::class)]
35#[UsesClass(\SqlFaker\Generation\Candidate\FixedLexemeGenerator::class)]
36#[UsesClass(\SqlFaker\Generation\Lexeme\Lexeme::class)]
37#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeCandidates::class)]
38#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeInput::class)]
39#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeSequence::class)]
40#[UsesClass(\SqlFaker\Generation\Candidate\MatchingLexemeGenerator::class)]
41#[UsesClass(\SqlFaker\Generation\Candidate\ValueLexemeGenerator::class)]
42#[UsesClass(\SqlFaker\Generation\Candidate\RegisteredLexemeGenerator::class)]
43#[UsesClass(\SqlFaker\Generation\Output\CandidateResolver::class)]
44#[UsesClass(\SqlFaker\Generation\Lexeme\OutputPart::class)]
45#[UsesClass(\SqlFaker\Generation\Lexeme\ResolvedOutput::class)]
46#[UsesClass(\SqlFaker\Generation\Output\ReverseLexemeGenerator::class)]
47#[UsesClass(\SqlFaker\Generation\Output\SqlSerializer::class)]
48#[UsesClass(\SqlFaker\Generation\Output\CombinedSpacingRule::class)]
49#[UsesClass(\SqlFaker\Generation\Lexeme\SpacingConstraint::class)]
50#[UsesClass(\SqlFaker\Generation\Token\TerminalOccurrence::class)]
51#[UsesClass(TerminalSequence::class)]
52#[UsesClass(\SqlFaker\Generation\Candidate\VersionCase::class)]
53#[UsesClass(\SqlFaker\Generation\Candidate\VersionedLexemeGenerator::class)]
54#[UsesClass(\SqlFaker\Sqlite\Generation\Lexeme\DefinitionFactory::class)]
55#[UsesClass(\SqlFaker\Sqlite\Generation\Lexeme\JoinLexemeGenerator::class)]
56#[UsesClass(\SqlFaker\Generation\Plan\ProductionPattern::class)]
57#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeBoundary::class)]
58#[UsesClass(\SqlFaker\Generation\Token\ProductionOccurrence::class)]
59#[UsesClass(\SqlFaker\Sqlite\Generation\Lexeme\JoinModifiers::class)]
60#[UsesClass(\SqlFaker\Sqlite\Generation\Lexeme\WindowNameLexemeGenerator::class)]
61#[UsesClass(\SqlFaker\Generation\Value\CharacterDomain::class)]
62#[UsesClass(\SqlFaker\Generation\Value\ChoiceDomain::class)]
63#[UsesClass(\SqlFaker\Generation\Value\IntegerDomain::class)]
64#[UsesClass(\SqlFaker\Generation\Value\SequenceDomain::class)]
65#[UsesClass(\SqlFaker\MySql\Generation\Value\DollarQuotedDomain::class)]
66#[UsesClass(\SqlFaker\Generation\Output\BoundaryCompletion::class)]
67#[UsesClass(\SqlFaker\Sqlite\Generation\Value\IdentifierDomain::class)]
68#[UsesClass(\SqlFaker\Sqlite\Generation\Value\QuotedDomain::class)]
69#[UsesClass(\SqlFaker\Generation\Value\RepeatDomain::class)]
70#[UsesClass(\SqlFaker\Generation\Value\WordDomain::class)]
71#[UsesClass(\SqlFaker\Generation\Value\ValueChoices::class)]
72#[UsesClass(\SqlFaker\Generation\Choice\BytePlanCompiler::class)]
73#[UsesClass(\SqlFaker\Sqlite\Generation\Lexeme\LexicalDefinition::class)]
74final class LexicalGrammarTest extends TestCase
75{
76 public function testGenerateQuotedIdentifierWritesWhatTheLexerReadsBackAsAnIdentifier(): void
77 {
78 $faker = Factory::create();
79 $faker->seed(12345);
80 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
81
82 self::assertSame(['ID'], $lexical->tokenize($lexical->generateQuotedIdentifier(3, 3)));
83 }
84
85 public function testGenerateStringLiteralWritesWhatTheLexerReadsBackAsAString(): void
86 {
87 $faker = Factory::create();
88 $faker->seed(12345);
89 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
90
91 self::assertSame(['STRING'], $lexical->tokenize($lexical->generateStringLiteral(3, 3)));
92 }
93
94 public function testGenerateIntegerLiteralWritesWhatTheLexerReadsBackAsAnInteger(): void
95 {
96 $faker = Factory::create();
97 $faker->seed(12345);
98 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
99
100 self::assertSame(['INTEGER'], $lexical->tokenize($lexical->generateIntegerLiteral(10, 10)));
101 }
102
103 public function testGenerateDecimalLiteralWritesWhatTheLexerReadsBackAsAFloat(): void
104 {
105 $faker = Factory::create();
106 $faker->seed(12345);
107 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
108
109 self::assertSame(['FLOAT'], $lexical->tokenize($lexical->generateDecimalLiteral(4, 2)));
110 }
111
112 public function testVersionReportsTheReleaseTheProfileWasBuiltFor(): void
113 {
114 self::assertSame('sqlite-3.47.2', (new LexicalGrammar(Factory::create(), 'sqlite-3.47.2'))->version());
115 }
116
117 /**
118 * @param Closure(LexicalGrammar): string $withDefaults
119 * @param Closure(LexicalGrammar): string $withExplicitBounds
120 */
121 #[DataProvider('providerPublicLexemeDefaults')]
122 public function testPublicLexemeDefaultBounds(Closure $withDefaults, Closure $withExplicitBounds): void
123 {
124 $faker = Factory::create();
125 $grammar = new LexicalGrammar($faker, 'sqlite-3.47.2');
126
127 $faker->seed(20_260_824);
128 $generated = $withDefaults($grammar);
129
130 $faker->seed(20_260_824);
131 $explicit = $withExplicitBounds($grammar);
132
133 self::assertNotSame('', $generated);
134 self::assertSame($generated, $explicit);
135 }
136
137 /**
138 * @return iterable<string, array{Closure(LexicalGrammar): string, Closure(LexicalGrammar): string}>
139 */
140 public static function providerPublicLexemeDefaults(): iterable
141 {
142 yield 'quoted identifier' => [
143 static fn (LexicalGrammar $grammar): string => $grammar->generateQuotedIdentifier(),
144 static fn (LexicalGrammar $grammar): string => $grammar->generateQuotedIdentifier(1, 128),
145 ];
146 yield 'string' => [
147 static fn (LexicalGrammar $grammar): string => $grammar->generateStringLiteral(),
148 static fn (LexicalGrammar $grammar): string => $grammar->generateStringLiteral(1, 255),
149 ];
150 yield 'integer' => [
151 static fn (LexicalGrammar $grammar): string => $grammar->generateIntegerLiteral(),
152 static fn (LexicalGrammar $grammar): string => $grammar->generateIntegerLiteral(1, PHP_INT_MAX),
153 ];
154 yield 'decimal' => [
155 static fn (LexicalGrammar $grammar): string => $grammar->generateDecimalLiteral(),
156 static fn (LexicalGrammar $grammar): string => $grammar->generateDecimalLiteral(15, 2),
157 ];
158 }
159
160 public function testTokenizesQuotedIdentifiersStringsVariablesAndComments(): void
161 {
162 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
163 $sql = <<<'SQL'
164SELECT "values", [select], `from`, 'FROM ''items''', X'af', ?, ?12, :name, @name, $name
165/* UPDATE */ -- DELETE
166FROM items
167SQL;
168
169 self::assertSame([
170 'SELECT', 'ID', 'COMMA', 'ID', 'COMMA', 'ID', 'COMMA', 'STRING', 'COMMA', 'BLOB', 'COMMA',
171 'VARIABLE', 'COMMA', 'VARIABLE', 'COMMA', 'VARIABLE', 'COMMA', 'VARIABLE', 'COMMA', 'VARIABLE',
172 'FROM', 'ID',
173 ], $lexical->tokenize($sql));
174 }
175
176 public function testUsesVersionedKeywordFamilies(): void
177 {
178 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
179
180 self::assertSame(['JOIN_KW', 'JOIN_KW', 'CTIME_KW'], $lexical->tokenize('LEFT CROSS CURRENT_TIMESTAMP'));
181 }
182
183 public function testTokenizesEveryNumberAndOperatorClass(): void
184 {
185 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
186
187 self::assertSame([
188 'INTEGER', 'QNUMBER', 'INTEGER', 'FLOAT', 'FLOAT', 'FLOAT', 'FLOAT',
189 'PTR', 'PTR', 'CONCAT', 'EQ', 'LE', 'NE', 'NE', 'GE', 'LSHIFT', 'RSHIFT',
190 'LP', 'RP', 'SEMI', 'COMMA', 'DOT', 'EQ', 'LT', 'GT', 'PLUS', 'MINUS', 'STAR',
191 'SLASH', 'REM', 'BITAND', 'BITOR', 'BITNOT',
192 ], $lexical->tokenize(
193 '0 1_0 0xAf 1.5 .5 1e2 1.e-2 '
194 . '->> -> || == <= <> != >= << >> ( ) ; , . = < > + - * / % & | ~',
195 ));
196 }
197
198 public function testTokenizesKeywordsCaseInsensitivelyAndSkipsEmbeddedComments(): void
199 {
200 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
201
202 self::assertSame(
203 ['SELECT', 'ID', 'FROM', 'ID'],
204 $lexical->tokenize("select-- comment\nname/* comment */from items"),
205 );
206 }
207
208 public function testRealizesStrictTableOptionAsIdentifierToken(): void
209 {
210 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
211
212 $sql = $lexical->realize([LexicalGrammar::STRICT_TABLE_OPTION]);
213 self::assertStringContainsString('STRICT', $sql);
214 self::assertSame(['ID'], $lexical->tokenize($sql));
215 self::assertSame(['ID'], $lexical->tokenize('STRICT'));
216 }
217
218 #[DataProvider('providerInvalidSql')]
219 public function testRejectsInvalidSql(string $sql, string $message): void
220 {
221 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
222
223 $this->expectException(LexicalException::class);
224 $this->expectExceptionMessage($message);
225
226 $lexical->tokenize($sql);
227 }
228
229 #[DataProvider('providerFixedTerminal')]
230 public function testRealizeOfFixedTerminal(string $terminal, string $expected): void
231 {
232 $faker = Factory::create();
233 $faker->seed(17);
234 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
235
236 self::assertSame($expected, $lexical->realize([$terminal]));
237 }
238
239 #[DataProvider('providerIdentifierTerminal')]
240 public function testRealizeOfIdentifier(string $terminal): void
241 {
242 $faker = Factory::create();
243 $faker->seed(17);
244 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
245
246 self::assertNotSame($terminal, $lexical->realize([$terminal]));
247 }
248
249 #[DataProvider('providerStringTerminal')]
250 public function testRealizeOfString(string $terminal): void
251 {
252 $faker = Factory::create();
253 $faker->seed(17);
254 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
255
256 self::assertStringStartsWith("'", $lexical->realize([$terminal]));
257 }
258
259 public function testRealizeOfGeneratedTerminals(): void
260 {
261 $faker = Factory::create();
262 $faker->seed(17);
263 $lexical = new LexicalGrammar($faker, 'sqlite-3.47.2');
264
265 self::assertMatchesRegularExpression("/^X'[0-9a-f]*'$/", $lexical->realize(['BLOB']));
266 self::assertMatchesRegularExpression('/^\d+$/', $lexical->realize(['number']));
267 self::assertMatchesRegularExpression('/^\d+$/', $lexical->realize(['INTEGER']));
268 self::assertMatchesRegularExpression('/^(?:\?\d*|[:@$][A-Za-z_][A-Za-z0-9_]*)$/', $lexical->realize(['VARIABLE']));
269 }
270
271 /**
272 * @return iterable<string, array{string, string}>
273 */
274 public static function providerInvalidSql(): iterable
275 {
276 yield 'bracket identifier' => ['[name', 'Unterminated SQLite bracket identifier.'];
277 yield 'single quoted string' => ["'value", 'Unterminated SQLite quoted token'];
278 yield 'double quoted identifier' => ['"name', 'Unterminated SQLite quoted token'];
279 yield 'backtick quoted identifier' => ['`name', 'Unterminated SQLite quoted token'];
280 yield 'block comment' => ['/* comment', 'Unterminated SQLite block comment.'];
281 yield 'unsupported character' => ['SELECT \\', 'offset 7: SELECT \\'];
282 }
283
284 /**
285 * @return iterable<string, array{string, string}>
286 */
287 public static function providerFixedTerminal(): iterable
288 {
289 yield 'QNUMBER' => ['QNUMBER', '1_0'];
290 yield 'ANY' => ['ANY', 'name'];
291 yield 'LP' => ['LP', '('];
292 yield 'RP' => ['RP', ')'];
293 yield 'SEMI' => ['SEMI', ';'];
294 yield 'COMMA' => ['COMMA', ','];
295 yield 'DOT' => ['DOT', '.'];
296 yield 'EQ' => ['EQ', '='];
297 yield 'LT' => ['LT', '<'];
298 yield 'PLUS' => ['PLUS', '+'];
299 yield 'MINUS' => ['MINUS', '-'];
300 yield 'STAR' => ['STAR', '*'];
301 yield 'BITAND' => ['BITAND', '&'];
302 yield 'BITNOT' => ['BITNOT', '~'];
303 yield 'CONCAT' => ['CONCAT', '||'];
304 yield 'PTR' => ['PTR', '->'];
305 }
306
307 /**
308 * @return iterable<string, array{string}>
309 */
310 public static function providerIdentifierTerminal(): iterable
311 {
312 yield 'ID' => ['ID'];
313 yield 'id' => ['id'];
314 yield 'idj' => ['idj'];
315 }
316
317 /**
318 * @return iterable<string, array{string}>
319 */
320 public static function providerStringTerminal(): iterable
321 {
322 yield 'ids' => ['ids'];
323 yield 'STRING' => ['STRING'];
324 }
325 public function testRealizeSequenceHonorsThePlannedStringWithoutRequiringTrivia(): void
326 {
327 $grammar = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
328 $plan = GenerationPlan::all()->withLexemes(['STRING' => ["'a''b'"]]);
329 self::assertSame("'a''b'", $grammar->realizeSequence(TerminalSequence::fromNames(['STRING']), $plan));
330 }
331
332 public function testIsNonOutputDoesNotConfuseOrdinaryValuesWithParserMarkers(): void
333 {
334 $grammar = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
335 self::assertFalse($grammar->isNonOutput('STRING'));
336 self::assertFalse($grammar->isNonOutput('UNIMPLEMENTED'));
337 }
338
339 public function testRealizeReportsAnUnimplementedTerminalAtItsActualUse(): void
340 {
341 $grammar = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
342 $this->expectException(LexicalException::class);
343 $this->expectExceptionMessage('UNIMPLEMENTED');
344 $grammar->realize(['UNIMPLEMENTED']);
345 }
346
347 public function testResolveSequenceExposesTheChosenCandidateAndItsOutput(): void
348 {
349 $lexical = new LexicalGrammar(Factory::create(), 'sqlite-3.47.2');
350 $output = $lexical->resolveSequence(TerminalSequence::fromNames(['INTEGER']), null, static fn (int $count): int => $count - 1);
351 self::assertSame('2', $output->parts[0]->lexeme->text);
352 self::assertCount(1, $output->candidates);
353 self::assertSame('', $output->parts[0]->separator);
354 }
355}
356