packages/sql-faker/tests/Unit/Generation/Derivation/TokenGeneratorTest.php

1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Generation\Derivation;
6
7use Faker\Factory;
8use PHPUnit\Framework\Attributes\CoversClass;
9use PHPUnit\Framework\Attributes\UsesClass;
10use PHPUnit\Framework\TestCase;
11use SqlFaker\Generation\Candidate\ChoiceLexemeGenerator;
12use SqlFaker\Generation\Candidate\FixedLexemeGenerator;
13use SqlFaker\Generation\Candidate\MatchingLexemeGenerator;
14use SqlFaker\Generation\Candidate\RegisteredLexemeGenerator;
15use SqlFaker\Generation\Candidate\SequenceLexemeGenerator;
16use SqlFaker\Generation\Candidate\ValueLexemeGenerator;
17use SqlFaker\Generation\Candidate\VersionCase;
18use SqlFaker\Generation\Candidate\VersionedLexemeGenerator;
19use SqlFaker\Generation\Derivation\CompletionCosts;
20use SqlFaker\Generation\Derivation\Derivation;
21use SqlFaker\Generation\Derivation\DerivationTrace;
22use SqlFaker\Generation\Derivation\TerminationAnalyzer;
23use SqlFaker\Generation\Derivation\TerminationCost;
24use SqlFaker\Generation\Derivation\TokenGenerator;
25use SqlFaker\Generation\Exception\GenerationException;
26use SqlFaker\Generation\Exception\LexicalException;
27use SqlFaker\Generation\Lexeme\Lexeme;
28use SqlFaker\Generation\Lexeme\LexemeBoundary;
29use SqlFaker\Generation\Lexeme\LexemeCandidates;
30use SqlFaker\Generation\Lexeme\LexemeGenerator;
31use SqlFaker\Generation\Lexeme\LexemeInput;
32use SqlFaker\Generation\Lexeme\LexemeSequence;
33use SqlFaker\Generation\Lexeme\OutputPart;
34use SqlFaker\Generation\Lexeme\ResolvedOutput;
35use SqlFaker\Generation\Lexeme\SpacingConstraint;
36use SqlFaker\Generation\Lexeme\SpacingRule;
37use SqlFaker\Generation\Output\CandidateResolver;
38use SqlFaker\Generation\Output\CombinedSpacingRule;
39use SqlFaker\Generation\Output\ReverseLexemeGenerator;
40use SqlFaker\Generation\Output\SqlSerializer;
41use SqlFaker\Generation\Plan\GenerationPlan;
42use SqlFaker\Generation\Plan\ProductionPattern;
43use SqlFaker\Generation\Plan\RulePlan;
44use SqlFaker\Generation\Token\ProductionOccurrence;
45use SqlFaker\Generation\Token\RewriteRule;
46use SqlFaker\Generation\Token\TerminalOccurrence;
47use SqlFaker\Generation\Token\TerminalSequence;
48use SqlFaker\Generation\Token\TokenRewriter;
49use SqlFaker\Grammar\Model\Grammar;
50use SqlFaker\Grammar\Model\NonTerminal;
51use SqlFaker\Grammar\Model\Production;
52use SqlFaker\Grammar\Model\ProductionRule;
53use SqlFaker\Grammar\Model\Terminal;
54use SqlFaker\MySql\Generation\Spacing\KeywordPhraseSpacingRule;
55
56#[CoversClass(TokenGenerator::class)]
57#[UsesClass(RewriteRule::class)]
58#[UsesClass(ProductionOccurrence::class)]
59#[UsesClass(TerminalSequence::class)]
60#[UsesClass(TokenRewriter::class)]
61#[UsesClass(TerminalOccurrence::class)]
62#[UsesClass(CandidateResolver::class)]
63#[UsesClass(ReverseLexemeGenerator::class)]
64#[UsesClass(ResolvedOutput::class)]
65#[UsesClass(OutputPart::class)]
66#[UsesClass(SqlSerializer::class)]
67#[UsesClass(VersionCase::class)]
68#[UsesClass(VersionedLexemeGenerator::class)]
69#[UsesClass(KeywordPhraseSpacingRule::class)]
70#[UsesClass(CombinedSpacingRule::class)]
71#[UsesClass(LexemeBoundary::class)]
72#[UsesClass(SpacingRule::class)]
73#[UsesClass(SpacingConstraint::class)]
74#[UsesClass(Lexeme::class)]
75#[UsesClass(LexemeGenerator::class)]
76#[UsesClass(ValueLexemeGenerator::class)]
77#[UsesClass(LexemeSequence::class)]
78#[UsesClass(RegisteredLexemeGenerator::class)]
79#[UsesClass(LexemeCandidates::class)]
80#[UsesClass(SequenceLexemeGenerator::class)]
81#[UsesClass(ChoiceLexemeGenerator::class)]
82#[UsesClass(FixedLexemeGenerator::class)]
83#[UsesClass(MatchingLexemeGenerator::class)]
84#[UsesClass(LexemeInput::class)]
85#[UsesClass(CompletionCosts::class)]
86#[UsesClass(Derivation::class)]
87#[UsesClass(DerivationTrace::class)]
88#[UsesClass(GenerationPlan::class)]
89#[UsesClass(TerminationAnalyzer::class)]
90#[UsesClass(TerminationCost::class)]
91#[UsesClass(Grammar::class)]
92#[UsesClass(NonTerminal::class)]
93#[UsesClass(Production::class)]
94#[UsesClass(ProductionRule::class)]
95#[UsesClass(Terminal::class)]
96#[UsesClass(GenerationException::class)]
97#[UsesClass(LexicalException::class)]
98#[UsesClass(ProductionPattern::class)]
99#[UsesClass(\SqlFaker\Generation\Output\BoundaryCompletion::class)]
100#[UsesClass(\SqlFaker\Generation\Derivation\CompletionState::class)]
101#[UsesClass(\SqlFaker\Generation\Derivation\CompletionFrontier::class)]
102#[UsesClass(\SqlFaker\Generation\Derivation\ConstrainedCompletion::class)]
103#[UsesClass(\SqlFaker\Generation\Value\ValueChoices::class)]
104#[UsesClass(\SqlFaker\Generation\Derivation\CompletionMemo::class)]
105#[UsesClass(\SqlFaker\Generation\Derivation\CompletionReduction::class)]
106#[UsesClass(\SqlFaker\Generation\Derivation\ConstraintDependencies::class)]
107#[UsesClass(\SqlFaker\Generation\Choice\BytePlanCompiler::class)]
108#[UsesClass(\SqlFaker\Generation\Derivation\Completion\PatternProductions::class)]
109#[UsesClass(\SqlFaker\Generation\Derivation\Completion\CompletionWitness::class)]
110#[UsesClass(RulePlan::class)]
111#[UsesClass(\SqlFaker\Generation\Derivation\DerivationNode::class)]
112#[UsesClass(\SqlFaker\Generation\Plan\Compilation\GrammarCompiler::class)]
113#[UsesClass(\SqlFaker\Generation\Plan\Compilation\Scope::class)]
114#[UsesClass(\SqlFaker\Generation\Plan\Compilation\PreparedGrammar::class)]
115#[UsesClass(\SqlFaker\Generation\Plan\Compilation\ScopedGeneration::class)]
116final class TokenGeneratorTest extends TestCase
117{
118    public function testGenerateCanChooseNullableChildrenWhenAnotherSiblingProvidesOutput(): void
119    {
120        $grammar = new Grammar('root', [
121            'root' => new ProductionRule('root', [new Production([new NonTerminal('optional'), new Terminal('UNKNOWN')])]),
122            'optional' => new ProductionRule('optional', [new Production([])]),
123        ]);
124        $generator = new TokenGenerator($grammar, Factory::create(), static fn (string $name): bool => false);
125        $sequence = $generator->generate('root', GenerationPlan::all()->requiringNonEmpty());
126        self::assertSame(['UNKNOWN'], $sequence->names());
127        self::assertCount(2, $sequence->productions);
128    }
129
130    public function testGenerateReservesVisibleOutputWithoutConsultingLexicalHandlerAvailability(): void
131    {
132        $grammar = new Grammar('root', [
133            'root' => new ProductionRule('root', [new Production([new Terminal('EOF')]), new Production([new Terminal('UNKNOWN')])]),
134        ]);
135        $generator = new TokenGenerator($grammar, Factory::create(), static fn (string $name): bool => $name === 'EOF');
136        self::assertSame(['UNKNOWN'], $generator->generate('root', GenerationPlan::all()->requiringNonEmpty())->names());
137    }
138
139    public function testDeriveRetainsOriginalRuleIdentityForRepeatedChoices(): void
140    {
141        $grammar = new Grammar('root', [
142            'root' => new ProductionRule('root', [new Production([new NonTerminal('value'), new NonTerminal('value')])]),
143            'value' => new ProductionRule('value', [new Production([new Terminal('ID')]), new Production([new Terminal('INTEGER')])]),
144        ]);
145        $generator = new TokenGenerator($grammar, Factory::create(), static fn (string $name): bool => false);
146        $plan = GenerationPlan::constrained('root', ['value' => [ProductionPattern::at(1), ProductionPattern::at(0)]]);
147        self::assertSame(['INTEGER', 'ID'], $generator->derive('root', $plan)->names());
148        $scoped = $plan->withRule('root', RulePlan::any()->withChild('value', 0, RulePlan::any()->allowing(ProductionPattern::exactly('INTEGER'))));
149        $sequence = $generator->generate('root', $scoped);
150        self::assertSame(['INTEGER', 'ID'], $sequence->names());
151        self::assertSame(['root', 'value', 'value'], array_column($sequence->productions, 'rule'));
152        self::assertSame([0, 1, 0], array_column($sequence->productions, 'ordinal'));
153    }
154
155    public function testMinimumExpansionsIncludesEveryPlannedListItem(): void
156    {
157        $grammar = new Grammar('list', [
158            'list' => new ProductionRule('list', [new Production([new NonTerminal('list'), new Terminal(','), new NonTerminal('item')]), new Production([new NonTerminal('item')])]),
159            'item' => new ProductionRule('item', [new Production([new Terminal('ID')]), new Production([new Terminal('INTEGER')])]),
160        ]);
161        $generator = new TokenGenerator($grammar, Factory::create(), static fn (string $name): bool => false);
162        $item = RulePlan::any()->withRule('item', RulePlan::any()->allowing(ProductionPattern::exactly('INTEGER')));
163        $plan = GenerationPlan::all()->withRule('list', RulePlan::any()->withItems($item, $item, $item));
164        self::assertSame(6, $generator->minimumExpansions('list', $plan));
165        self::assertSame(['INTEGER', ',', 'INTEGER', ',', 'INTEGER'], $generator->generate('list', $plan->withExpansionBudget(6))->names());
166    }
167
168    public function testPrepareKeepsPlansAndUnconstrainedCallsIndependent(): void
169    {
170        $grammar = new Grammar('root', ['root' => new ProductionRule('root', [new Production([new Terminal('A')]), new Production([new Terminal('B')])])]);
171        $generator = new TokenGenerator($grammar, Factory::create(), static fn (string $name): bool => false);
172        $a = GenerationPlan::all()->withRule('root', RulePlan::any()->allowing(ProductionPattern::exactly('A')));
173        $b = GenerationPlan::all()->withRule('root', RulePlan::any()->allowing(ProductionPattern::exactly('B')));
174        $generator->prepare('root', $a);
175        self::assertSame(['A'], $generator->generate('root', $a)->names());
176        self::assertSame(['A'], $generator->generate('root', $a)->names());
177        self::assertSame(['B'], $generator->generate('root', GenerationPlan::all(), static fn (int $count): int => $count - 1)->names());
178        self::assertNull($generator->lastScope);
179        self::assertSame(['B'], $generator->generate('root', $b)->names());
180        self::assertSame(['A'], $generator->generate('root', $a)->names());
181    }
182
183
184    public function testMinimumExpansionsUsesTheCurrentRootPlanAndExplicitBudget(): void
185    {
186        $grammar = new Grammar('root', [
187            'root' => new ProductionRule('root', [new Production([new Terminal('A')]), new Production([new NonTerminal('leaf')])]),
188            'leaf' => new ProductionRule('leaf', [new Production([new Terminal('B')])]),
189        ]);
190        $generator = new TokenGenerator($grammar, Factory::create(), static fn (string $name): bool => false);
191        $scoped = GenerationPlan::all()->withRule('root', RulePlan::any()->allowing(ProductionPattern::exactly('leaf')));
192        self::assertSame(2, $generator->minimumExpansions('root', $scoped));
193        self::assertSame(1, $generator->minimumExpansions('leaf', $scoped));
194        self::assertSame(1, $generator->minimumExpansions('root', GenerationPlan::all()));
195        self::assertSame(PHP_INT_MAX, $generator->minimumExpansions('root', $scoped->withPatternForEveryOccurrence('leaf', ProductionPattern::exactly('B'))->withExpansionBudget(1)));
196    }
197
198}
199