packages/sql-faker/tests/Unit/PostgreSql/Generation/LexicalGrammarTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\SqlFaker\PostgreSql\Generation;
6
7use Closure;
8use Faker\Factory;
9use PHPUnit\Framework\Attributes\CoversClass;
10use PHPUnit\Framework\Attributes\DataProvider;
11use PHPUnit\Framework\Attributes\UsesClass;
12use PHPUnit\Framework\TestCase;
13use SqlFaker\Generation\Exception\LexicalException;
14use SqlFaker\Generation\Plan\GenerationPlan;
15use SqlFaker\Generation\Token\TerminalSequence;
16use SqlFaker\Generation\Value\RandomCharacters;
17use SqlFaker\Grammar\Resource\SqlVersion;
18use SqlFaker\Grammar\Resource\SqlVersionRegistry;
19use SqlFaker\PostgreSql\Generation\LexicalGrammar;
20use SqlFaker\PostgreSql\Generation\Lookahead\PgLookahead;
21use SqlFaker\PostgreSql\Generation\Tokenization\KeywordIndex;
22use SqlFaker\PostgreSql\Generation\Tokenization\PgTokenizer;
23use SqlFaker\PostgreSql\Generation\Value\LiteralGenerator;
24
25#[CoversClass(LexicalGrammar::class)]
26#[CoversClass(LiteralGenerator::class)]
27#[UsesClass(SqlVersion::class)]
28#[UsesClass(LexicalException::class)]
29#[UsesClass(KeywordIndex::class)]
30#[UsesClass(RandomCharacters::class)]
31#[UsesClass(SqlVersionRegistry::class)]
32#[UsesClass(PgLookahead::class)]
33#[UsesClass(PgTokenizer::class)]
34#[UsesClass(\SqlFaker\PostgreSql\Generation\Tokenization\PgQuoting::class)]
35#[UsesClass(\SqlFaker\Generation\Candidate\ChoiceLexemeGenerator::class)]
36#[UsesClass(\SqlFaker\Generation\Candidate\FixedLexemeGenerator::class)]
37#[UsesClass(\SqlFaker\Generation\Lexeme\Lexeme::class)]
38#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeCandidates::class)]
39#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeInput::class)]
40#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeSequence::class)]
41#[UsesClass(\SqlFaker\Generation\Candidate\MatchingLexemeGenerator::class)]
42#[UsesClass(\SqlFaker\Generation\Candidate\ValueLexemeGenerator::class)]
43#[UsesClass(\SqlFaker\Generation\Candidate\RegisteredLexemeGenerator::class)]
44#[UsesClass(\SqlFaker\Generation\Output\CandidateResolver::class)]
45#[UsesClass(\SqlFaker\Generation\Lexeme\OutputPart::class)]
46#[UsesClass(\SqlFaker\Generation\Lexeme\ResolvedOutput::class)]
47#[UsesClass(\SqlFaker\Generation\Output\ReverseLexemeGenerator::class)]
48#[UsesClass(\SqlFaker\Generation\Output\SqlSerializer::class)]
49#[UsesClass(\SqlFaker\Generation\Output\CombinedSpacingRule::class)]
50#[UsesClass(\SqlFaker\Generation\Lexeme\LexemeBoundary::class)]
51#[UsesClass(\SqlFaker\Generation\Lexeme\SpacingConstraint::class)]
52#[UsesClass(\SqlFaker\Generation\Token\TerminalOccurrence::class)]
53#[UsesClass(TerminalSequence::class)]
54#[UsesClass(\SqlFaker\Generation\Candidate\VersionCase::class)]
55#[UsesClass(\SqlFaker\Generation\Candidate\VersionedLexemeGenerator::class)]
56#[UsesClass(\SqlFaker\PostgreSql\Generation\Lexeme\DefinitionFactory::class)]
57#[UsesClass(\SqlFaker\PostgreSql\Generation\Lexeme\HashBoundLexemeGenerator::class)]
58#[UsesClass(\SqlFaker\PostgreSql\Generation\Lexeme\KeywordLexemeGenerator::class)]
59#[UsesClass(GenerationPlan::class)]
60#[UsesClass(\SqlFaker\Generation\Plan\ProductionPattern::class)]
61#[UsesClass(\SqlFaker\Generation\Token\ProductionOccurrence::class)]
62#[UsesClass(\SqlFaker\Generation\Value\CharacterDomain::class)]
63#[UsesClass(\SqlFaker\Generation\Value\ChoiceDomain::class)]
64#[UsesClass(\SqlFaker\Generation\Value\IntegerDomain::class)]
65#[UsesClass(\SqlFaker\Generation\Value\SequenceDomain::class)]
66#[UsesClass(\SqlFaker\PostgreSql\Generation\Value\DollarQuotedDomain::class)]
67#[UsesClass(\SqlFaker\Generation\Output\BoundaryCompletion::class)]
68#[UsesClass(\SqlFaker\Generation\Candidate\IntegerLexemeGenerator::class)]
69#[UsesClass(\SqlFaker\PostgreSql\Generation\Value\IdentifierDomain::class)]
70#[UsesClass(\SqlFaker\PostgreSql\Generation\Value\OperatorDomain::class)]
71#[UsesClass(\SqlFaker\PostgreSql\Generation\Value\QuotedDomain::class)]
72#[UsesClass(\SqlFaker\Generation\Value\WordDomain::class)]
73#[UsesClass(\SqlFaker\Generation\Value\ValueChoices::class)]
74#[UsesClass(\SqlFaker\Generation\Choice\BytePlanCompiler::class)]
75#[UsesClass(\SqlFaker\PostgreSql\Generation\Lexeme\LexicalDefinition::class)]
76final class LexicalGrammarTest extends TestCase
77{
78 public function testGenerateQuotedIdentifierWritesWhatTheLexerReadsBackAsAnIdentifier(): void
79 {
80 $faker = Factory::create();
81 $faker->seed(12345);
82 $lexical = new LexicalGrammar($faker, 'pg-17.2');
83
84 self::assertSame(['IDENT'], $lexical->tokenize($lexical->generateQuotedIdentifier(3, 3)));
85 }
86
87 public function testGenerateStringLiteralWritesWhatTheLexerReadsBackAsAString(): void
88 {
89 $faker = Factory::create();
90 $faker->seed(12345);
91 $lexical = new LexicalGrammar($faker, 'pg-17.2');
92
93 self::assertSame(['SCONST'], $lexical->tokenize($lexical->generateStringLiteral(3, 3)));
94 }
95
96 public function testGenerateIntegerLiteralWritesWhatTheLexerReadsBackAsAnInteger(): void
97 {
98 $faker = Factory::create();
99 $faker->seed(12345);
100 $lexical = new LexicalGrammar($faker, 'pg-17.2');
101
102 self::assertSame(['ICONST'], $lexical->tokenize($lexical->generateIntegerLiteral(10, 10)));
103 }
104
105 public function testGenerateDecimalLiteralWritesWhatTheLexerReadsBackAsAFloat(): void
106 {
107 $faker = Factory::create();
108 $faker->seed(12345);
109 $lexical = new LexicalGrammar($faker, 'pg-17.2');
110
111 self::assertSame(['FCONST'], $lexical->tokenize($lexical->generateDecimalLiteral(4, 2)));
112 }
113
114 public function testGenerateFloatLiteralWritesWhatTheLexerReadsBackAsAFloat(): void
115 {
116 $faker = Factory::create();
117 $faker->seed(12345);
118 $lexical = new LexicalGrammar($faker, 'pg-17.2');
119
120 self::assertSame(['FCONST'], $lexical->tokenize($lexical->generateFloatLiteral(4, 2, 2, 2)));
121 }
122
123 public function testGenerateHexLiteralWritesWhatTheLexerReadsBackAsAHexString(): void
124 {
125 $faker = Factory::create();
126 $faker->seed(12345);
127 $lexical = new LexicalGrammar($faker, 'pg-17.2');
128
129 self::assertSame(['XCONST'], $lexical->tokenize($lexical->generateHexLiteral(4, 4)));
130 }
131
132 public function testGenerateBinaryLiteralWritesWhatTheLexerReadsBackAsABitString(): void
133 {
134 $faker = Factory::create();
135 $faker->seed(12345);
136 $lexical = new LexicalGrammar($faker, 'pg-17.2');
137
138 self::assertSame(['BCONST'], $lexical->tokenize($lexical->generateBinaryLiteral(4, 4)));
139 }
140
141 public function testGenerateDollarQuotedStringWritesWhatTheLexerReadsBackAsAString(): void
142 {
143 $faker = Factory::create();
144 $faker->seed(12345);
145 $lexical = new LexicalGrammar($faker, 'pg-17.2');
146
147 self::assertSame(['SCONST'], $lexical->tokenize($lexical->generateDollarQuotedString(3, 3)));
148 }
149
150 public function testGenerateParameterMarkerWritesWhatTheLexerReadsBackAsAParameter(): void
151 {
152 $faker = Factory::create();
153 $faker->seed(12345);
154 $lexical = new LexicalGrammar($faker, 'pg-17.2');
155
156 self::assertSame(['PARAM'], $lexical->tokenize($lexical->generateParameterMarker(2, 2)));
157 }
158
159 public function testVersionReportsTheReleaseTheProfileWasBuiltFor(): void
160 {
161 self::assertSame('pg-17.2', (new LexicalGrammar(Factory::create(), 'pg-17.2'))->version());
162 }
163
164 /**
165 * @param Closure(LexicalGrammar): string $withDefaults
166 * @param Closure(LexicalGrammar): string $withExplicitBounds
167 */
168 #[DataProvider('providerPublicLexemeDefaults')]
169 public function testPublicLexemeDefaultBounds(Closure $withDefaults, Closure $withExplicitBounds): void
170 {
171 $faker = Factory::create();
172 $grammar = new LexicalGrammar($faker, 'pg-17.2');
173
174 $faker->seed(20_260_824);
175 $generated = $withDefaults($grammar);
176
177 $faker->seed(20_260_824);
178 $explicit = $withExplicitBounds($grammar);
179
180 self::assertNotSame('', $generated);
181 self::assertSame($generated, $explicit);
182 }
183
184 /**
185 * @return iterable<string, array{Closure(LexicalGrammar): string, Closure(LexicalGrammar): string}>
186 */
187 public static function providerPublicLexemeDefaults(): iterable
188 {
189 yield 'quoted identifier' => [
190 static fn (LexicalGrammar $grammar): string => $grammar->generateQuotedIdentifier(),
191 static fn (LexicalGrammar $grammar): string => $grammar->generateQuotedIdentifier(1, 63),
192 ];
193 yield 'string' => [
194 static fn (LexicalGrammar $grammar): string => $grammar->generateStringLiteral(),
195 static fn (LexicalGrammar $grammar): string => $grammar->generateStringLiteral(1, 255),
196 ];
197 yield 'integer' => [
198 static fn (LexicalGrammar $grammar): string => $grammar->generateIntegerLiteral(),
199 static fn (LexicalGrammar $grammar): string => $grammar->generateIntegerLiteral(1, 2147483647),
200 ];
201 yield 'decimal' => [
202 static fn (LexicalGrammar $grammar): string => $grammar->generateDecimalLiteral(),
203 static fn (LexicalGrammar $grammar): string => $grammar->generateDecimalLiteral(10, 2),
204 ];
205 yield 'float' => [
206 static fn (LexicalGrammar $grammar): string => $grammar->generateFloatLiteral(),
207 static fn (LexicalGrammar $grammar): string => $grammar->generateFloatLiteral(10, 2, -307, 308),
208 ];
209 yield 'hex' => [
210 static fn (LexicalGrammar $grammar): string => $grammar->generateHexLiteral(),
211 static fn (LexicalGrammar $grammar): string => $grammar->generateHexLiteral(1, 16),
212 ];
213 yield 'binary' => [
214 static fn (LexicalGrammar $grammar): string => $grammar->generateBinaryLiteral(),
215 static fn (LexicalGrammar $grammar): string => $grammar->generateBinaryLiteral(1, 64),
216 ];
217 yield 'dollar quoted string' => [
218 static fn (LexicalGrammar $grammar): string => $grammar->generateDollarQuotedString(),
219 static fn (LexicalGrammar $grammar): string => $grammar->generateDollarQuotedString(1, 255),
220 ];
221 yield 'parameter marker' => [
222 static fn (LexicalGrammar $grammar): string => $grammar->generateParameterMarker(),
223 static fn (LexicalGrammar $grammar): string => $grammar->generateParameterMarker(1, 99),
224 ];
225 }
226
227 public function testTokenizesAllProblematicLiteralAndOperatorFamilies(): void
228 {
229 $lexical = new LexicalGrammar(Factory::create(), 'pg-17.2');
230 $sql = <<<'SQL'
231SELECT "values", 'a''b', E'a\b', $$FROM ?$$, $tag$WHERE$tag$, B'101', X'af', $1, data ?| tags
232/* UPDATE */ -- DELETE
233FROM items
234SQL;
235
236 self::assertSame([
237 'SELECT', 'IDENT', ',', 'SCONST', ',', 'SCONST', ',', 'SCONST', ',', 'SCONST', ',', 'BCONST', ',',
238 'XCONST', ',', 'PARAM', ',', 'DATA_P', 'Op', 'IDENT', 'FROM', 'IDENT',
239 ], $lexical->tokenize($sql));
240 }
241
242 public function testAppliesParserLookaheadOnlyInItsVersionedContext(): void
243 {
244 $lexical = new LexicalGrammar(Factory::create(), 'pg-17.2');
245
246 self::assertSame(['WITH_LA', 'TIME'], $lexical->tokenize('WITH TIME'));
247 self::assertSame(['WITH', 'RETURNS'], $lexical->tokenize('WITH RETURNS'));
248 }
249
250 public function testNormalizeLookaheadSettlesDerivedTokensFromTheirFollowers(): void
251 {
252 $lexical = new LexicalGrammar(Factory::create(), 'pg-17.2');
253
254 self::assertSame(
255 ['WITH', 'IDENT', 'WITH_LA', 'TIME', 'FORMAT', 'IDENT', 'FORMAT_LA', 'JSON'],
256 $lexical->normalizeLookahead([
257 'WITH_LA', 'IDENT', 'WITH', 'TIME', 'FORMAT_LA', 'IDENT', 'FORMAT', 'JSON',
258 ]),
259 );
260 }
261
262 public function testTokenizesCommentsAdjacentToOperators(): void
263 {
264 $lexical = new LexicalGrammar(Factory::create(), 'pg-17.2');
265 $sql = "SELECT a=/* outer /* inner */ outer */b, c/-- line\nd, e///* block */f";
266
267 self::assertSame(
268 ['SELECT', 'IDENT', '=', 'IDENT', ',', 'IDENT', '/', 'IDENT', ',', 'IDENT', 'Op', 'IDENT'],
269 $lexical->tokenize($sql),
270 );
271 }
272
273 public function testRealizesLookaheadTokenWithRequiredFollower(): void
274 {
275 $lexical = new LexicalGrammar(Factory::create(), 'pg-17.2');
276 $sql = $lexical->realize(['WITH_LA', 'TIME', 'ZONE']);
277
278 self::assertSame(['WITH_LA', 'TIME', 'ZONE'], $lexical->tokenize($sql));
279 }
280
281 public function testRejectsLookaheadTokenWithoutRequiredFollower(): void
282 {
283 $this->expectException(LexicalException::class);
284 $this->expectExceptionMessage('No compatible lexeme for WITH_LA');
285
286 (new LexicalGrammar(Factory::create(), 'pg-17.2'))->realize(['WITH_LA', 'RETURNS']);
287 }
288 public function testRealizeSequenceHonorsThePlannedStringWithoutRequiringTrivia(): void
289 {
290 $grammar = new LexicalGrammar(Factory::create(), 'pg-17.2');
291 $plan = GenerationPlan::all()->withLexemes(['SCONST' => ["'a''b'"]]);
292 self::assertSame("'a''b'", $grammar->realizeSequence(TerminalSequence::fromNames(['SCONST']), $plan));
293 }
294
295 public function testIsNonOutputDoesNotConfuseOrdinaryValuesWithParserMarkers(): void
296 {
297 $grammar = new LexicalGrammar(Factory::create(), 'pg-17.2');
298 self::assertFalse($grammar->isNonOutput('SCONST'));
299 self::assertFalse($grammar->isNonOutput('UNIMPLEMENTED'));
300 }
301
302 public function testRealizeReportsAnUnimplementedTerminalAtItsActualUse(): void
303 {
304 $grammar = new LexicalGrammar(Factory::create(), 'pg-17.2');
305 $this->expectException(LexicalException::class);
306 $this->expectExceptionMessage('UNIMPLEMENTED');
307 $grammar->realize(['UNIMPLEMENTED']);
308 }
309
310 public function testResolveSequenceExposesTheChosenCandidateAndItsOutput(): void
311 {
312 $lexical = new LexicalGrammar(Factory::create(), 'pg-17.2');
313 $output = $lexical->resolveSequence(TerminalSequence::fromNames(['ICONST']), null, static fn (int $count): int => $count - 1);
314 self::assertSame('1_0', $output->parts[0]->lexeme->text);
315 self::assertCount(1, $output->candidates);
316 self::assertSame('', $output->parts[0]->separator);
317 }
318}
319