packages/requirements/tests/Unit/Verification/VerifierTest.php

1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Verification;
6
7use JsonException;
8use PHPUnit\Framework\Attributes\CoversClass;
9use PHPUnit\Framework\Attributes\DataProvider;
10use PHPUnit\Framework\Attributes\Small;
11use PHPUnit\Framework\Attributes\UsesClass;
12use PHPUnit\Framework\TestCase;
13use Requirements\Config\CoverageThresholds;
14use Requirements\Config\DefinitionReader;
15use Requirements\Config\Definitions;
16use Requirements\Config\DocumentReader;
17use Requirements\Config\ExtensionClasses;
18use Requirements\Config\JsonSchemaFile;
19use Requirements\Config\LinkValidator;
20use Requirements\Config\Loader;
21use Requirements\Config\MarkdownDocument;
22use Requirements\Config\SchemaValidator;
23use Requirements\Ears\ConditionOrder;
24use Requirements\Ears\LiteralMask;
25use Requirements\Ears\SystemResponse;
26use Requirements\Ears\Validator;
27use Requirements\Ears\Wording;
28use Requirements\Input\Fields;
29use Requirements\Input\InvalidInputException;
30use Requirements\Model\Excerpt;
31use Requirements\Model\Item;
32use Requirements\Model\ItemValidator;
33use Requirements\Model\Project;
34use Requirements\Model\Source;
35use Requirements\Model\TestReference;
36use Requirements\Report\Analysis;
37use Requirements\Report\Analyzer;
38use Requirements\Report\Claims;
39use Requirements\Report\EvidenceMatcher;
40use Requirements\Report\SourceUnit;
41use Requirements\Report\UnitCollector;
42use Requirements\Source\DomSource;
43use Requirements\Source\JsonSource;
44use Requirements\Source\Registry as SourceRegistry;
45use Requirements\Source\ResourceLoader;
46use Requirements\Source\TextSource;
47use Requirements\Source\Unit;
48use Requirements\Test\BehatRunner;
49use Requirements\Test\PhpUnitRunner;
50use Requirements\Test\Registry;
51use Requirements\Test\RunnerConfig;
52use Requirements\Test\TestResult;
53use Requirements\Verification\TargetResults;
54use Requirements\Verification\TestExecution;
55use Requirements\Verification\VerificationResult;
56use Requirements\Verification\Verifier;
57use stdClass;
58use Tests\Fake\CountingRunner;
59use Tests\Fake\MemorySource;
60use Tests\Fake\ProjectDirectory;
61
62#[CoversClass(Verifier::class)]
63#[UsesClass(TestExecution::class)]
64#[UsesClass(TargetResults::class)]
65#[UsesClass(VerificationResult::class)]
66#[UsesClass(Registry::class)]
67#[UsesClass(PhpUnitRunner::class)]
68#[UsesClass(BehatRunner::class)]
69#[UsesClass(RunnerConfig::class)]
70#[UsesClass(TestResult::class)]
71#[UsesClass(Project::class)]
72#[UsesClass(Item::class)]
73#[UsesClass(TestReference::class)]
74#[UsesClass(Loader::class)]
75#[UsesClass(Analyzer::class)]
76#[UsesClass(CoverageThresholds::class)]
77#[UsesClass(DefinitionReader::class)]
78#[UsesClass(Definitions::class)]
79#[UsesClass(DocumentReader::class)]
80#[UsesClass(ExtensionClasses::class)]
81#[UsesClass(JsonSchemaFile::class)]
82#[UsesClass(LinkValidator::class)]
83#[UsesClass(MarkdownDocument::class)]
84#[UsesClass(SchemaValidator::class)]
85#[UsesClass(ConditionOrder::class)]
86#[UsesClass(LiteralMask::class)]
87#[UsesClass(SystemResponse::class)]
88#[UsesClass(Validator::class)]
89#[UsesClass(Wording::class)]
90#[UsesClass(Fields::class)]
91#[UsesClass(Excerpt::class)]
92#[UsesClass(ItemValidator::class)]
93#[UsesClass(Source::class)]
94#[UsesClass(Analysis::class)]
95#[UsesClass(Claims::class)]
96#[UsesClass(EvidenceMatcher::class)]
97#[UsesClass(SourceUnit::class)]
98#[UsesClass(UnitCollector::class)]
99#[UsesClass(DomSource::class)]
100#[UsesClass(JsonSource::class)]
101#[UsesClass(ResourceLoader::class)]
102#[UsesClass(TextSource::class)]
103#[UsesClass(Unit::class)]
104#[UsesClass(SourceRegistry::class)]
105#[Small]
106final class VerifierTest extends TestCase
107{
108    /**
109     * @throws JsonException
110     */
111    public function testVerifyRunsCustomSourcesAndRunnersThroughTheSamePipelineAndDeduplicatesTests(): void
112    {
113        $project = new ProjectDirectory();
114        $project->write('requirements.yaml', [
115            'version' => 1,
116            'definitions' => ['definition.yaml'],
117            'extensions' => ['sources' => ['service' => MemorySource::class], 'runners' => ['custom' => CountingRunner::class]],
118            'runners' => ['example' => ['extension' => 'custom', 'command' => ['example']]],
119        ]);
120        $item = ['id' => 'SPEC-1', 'statement' => 'The service shall retain its message.', 'evidence' => [['selector' => 'message:1', 'quote' => 'A service message.']], 'tests' => [['runner' => 'example', 'target' => 'shared']]];
121        $second = [...$item, 'id' => 'SPEC-2'];
122        $unsupported = [...$item, 'id' => 'SPEC-3', 'status' => 'unsupported', 'reason' => 'The external system owns this behavior.', 'tests' => [['runner' => 'example', 'target' => 'never']]];
123        $project->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'service', 'uri' => 'service:channel', 'format' => 'service', 'selector' => '*'], 'items' => [$item, $second, $unsupported]]);
124        $loaded = (new Loader())->load($project->directory . '/requirements.yaml');
125        $analysis = (new Analyzer())->analyze($loaded);
126        self::assertSame([], $analysis->errors);
127        self::assertSame(100.0, $analysis->summary()['percentage']);
128        $results = (new Verifier())->verify($loaded, $loaded->items);
129        self::assertSame('passed', $results['SPEC-1']->status);
130        self::assertSame('passed', $results['SPEC-2']->status);
131        self::assertSame(1, $results['SPEC-1']->passedTargets);
132        self::assertSame(1, $results['SPEC-2']->totalTargets);
133        self::assertSame('unsupported', $results['SPEC-3']->status);
134        self::assertSame("shared\n", $project->read('executions.txt'));
135    }
136
137    /**
138     * @throws JsonException
139     */
140    public function testVerifyDoesNotCountAZeroCaseSuccessAsAPassingTarget(): void
141    {
142        $project = new ProjectDirectory();
143        $project->write('requirements.yaml', [
144            'version' => 1,
145            'definitions' => ['definition.yaml'],
146            'extensions' => ['runners' => ['custom' => CountingRunner::class]],
147            'runners' => ['example' => ['extension' => 'custom', 'command' => ['example']]],
148        ]);
149        $project->write('definition.yaml', ['version' => 1, 'source' => null, 'items' => [[
150            'id' => 'SPEC-1', 'statement' => 'The reader shall retain positions.', 'origin' => 'original', 'reason' => 'Support editors.',
151            'tests' => [['runner' => 'example', 'target' => 'empty']],
152        ]]]);
153        $loaded = (new Loader())->load($project->directory . '/requirements.yaml');
154        $result = (new Verifier())->verify($loaded, $loaded->items)['SPEC-1'];
155        self::assertSame('failed', $result->status);
156        self::assertSame(0, $result->passedTargets);
157        self::assertSame(1, $result->totalTargets);
158        self::assertSame(0, $result->tests);
159    }
160
161    public function testVerifyReportsRequirementsAsNotApplicableWithoutRunningThem(): void
162    {
163        $directory = new ProjectDirectory();
164        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
165        $item = new Item('REQ-1', 'requirement', 'Names start with letters', 'unsupported', null, [], [new TestReference('example', 'a'), new TestReference('example', 'a'), new TestReference('example', 'b')], [], [], [], '', 'sourced', '', 'definition.yaml', []);
166        $results = (new Verifier())->verify($project, ['REQ-1' => $item]);
167        self::assertSame(['status' => 'not-applicable', 'tests' => 0, 'passed_targets' => null, 'total_targets' => 2, 'message' => '', 'deferred_targets' => 0], $results['REQ-1']->toArray());
168        self::assertFileDoesNotExist($directory->path('executions.txt'));
169    }
170
171    public function testVerifyReportsUnsupportedSpecificationsWithoutRunningThem(): void
172    {
173        $directory = new ProjectDirectory();
174        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
175        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'unsupported', null, [], [new TestReference('example', 'a')], [], [], [], '', 'original', 'Owned elsewhere.', 'definition.yaml', []);
176        $results = (new Verifier())->verify($project, ['SPEC-1' => $item], true);
177        self::assertSame(['status' => 'unsupported', 'tests' => 0, 'passed_targets' => null, 'total_targets' => 1, 'message' => '', 'deferred_targets' => 0], $results['SPEC-1']->toArray());
178        self::assertFileDoesNotExist($directory->path('executions.txt'));
179    }
180
181    public function testVerifyReportsLinkedTargetsAsNotRunWithoutTests(): void
182    {
183        $directory = new ProjectDirectory();
184        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
185        $linked = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'a'), new TestReference('example', 'b')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
186        $unlinked = new Item('SPEC-2', 'specification', 'The reader shall read.', 'supported', null, [], [], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
187        $results = (new Verifier())->verify($project, ['SPEC-1' => $linked, 'SPEC-2' => $unlinked], true);
188        self::assertSame(['status' => 'not-run', 'tests' => 0, 'passed_targets' => null, 'total_targets' => 2, 'message' => '', 'deferred_targets' => 0], $results['SPEC-1']->toArray());
189        self::assertSame(['status' => 'not-run', 'tests' => 0, 'passed_targets' => null, 'total_targets' => 0, 'message' => '', 'deferred_targets' => 0], $results['SPEC-2']->toArray());
190        self::assertFileDoesNotExist($directory->path('executions.txt'));
191    }
192
193    public function testVerifyReportsSpecificationsWithoutTestsAsUnverified(): void
194    {
195        $directory = new ProjectDirectory();
196        $project = new Project($directory->directory, [], [], [], [], [], 0.0, 0.0, [], []);
197        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
198        $results = (new Verifier())->verify($project, ['SPEC-1' => $item]);
199        self::assertSame(['status' => 'unverified', 'tests' => 0, 'passed_targets' => 0, 'total_targets' => 0, 'message' => 'No tests linked.', 'deferred_targets' => 0], $results['SPEC-1']->toArray());
200    }
201
202    public function testVerifyPassesWhenEveryTargetPasses(): void
203    {
204        $directory = new ProjectDirectory();
205        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
206        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'a'), new TestReference('example', 'b')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
207        $results = (new Verifier())->verify($project, ['SPEC-1' => $item]);
208        self::assertSame(['status' => 'passed', 'tests' => 2, 'passed_targets' => 2, 'total_targets' => 2, 'message' => '', 'deferred_targets' => 0], $results['SPEC-1']->toArray());
209        self::assertSame("a\nb\n", $directory->read('executions.txt'));
210    }
211
212    public function testVerifyFailsWhenAnyTargetFails(): void
213    {
214        $directory = new ProjectDirectory();
215        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory), 'unit' => new RunnerConfig('phpunit', [PHP_BINARY], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
216        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('unit', 'malformed'), new TestReference('example', 'a'), new TestReference('example', 'empty')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
217        $results = (new Verifier())->verify($project, ['SPEC-1' => $item]);
218        self::assertSame(['status' => 'failed', 'tests' => 1, 'passed_targets' => 1, 'total_targets' => 3, 'message' => "malformed: PHPUnit targets must be fully qualified Class::method references.\nempty: ", 'deferred_targets' => 0], $results['SPEC-1']->toArray());
219        self::assertSame("a\nempty\n", $directory->read('executions.txt'));
220    }
221
222    public function testVerifyRunsASharedTargetOnce(): void
223    {
224        $directory = new ProjectDirectory();
225        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
226        $first = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'shared'), new TestReference('example', 'shared')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
227        $second = new Item('SPEC-2', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'shared'), new TestReference('example', 'own')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
228        $results = (new Verifier())->verify($project, ['SPEC-1' => $first, 'SPEC-2' => $second]);
229        self::assertSame(['status' => 'passed', 'tests' => 1, 'passed_targets' => 1, 'total_targets' => 1, 'message' => '', 'deferred_targets' => 0], $results['SPEC-1']->toArray());
230        self::assertSame(['status' => 'passed', 'tests' => 2, 'passed_targets' => 2, 'total_targets' => 2, 'message' => '', 'deferred_targets' => 0], $results['SPEC-2']->toArray());
231        self::assertSame("shared\nown\n", $directory->read('executions.txt'));
232    }
233
234    public function testVerifyRunsTheSameTargetOfDifferentRunnersSeparately(): void
235    {
236        $directory = new ProjectDirectory();
237        $project = new Project($directory->directory, [], [], ['first' => new RunnerConfig('custom', ['first'], $directory->directory), 'second' => new RunnerConfig('custom', ['second'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
238        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('first', 'a'), new TestReference('second', 'a')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
239        $results = (new Verifier())->verify($project, ['SPEC-1' => $item]);
240        self::assertSame(['status' => 'passed', 'tests' => 2, 'passed_targets' => 2, 'total_targets' => 2, 'message' => '', 'deferred_targets' => 0], $results['SPEC-1']->toArray());
241        self::assertSame("a\na\n", $directory->read('executions.txt'));
242    }
243
244    public function testVerifyKeysResultsByItemId(): void
245    {
246        $directory = new ProjectDirectory();
247        $project = new Project($directory->directory, [], [], [], [], [], 0.0, 0.0, [], []);
248        $requirement = new Item('REQ-1', 'requirement', 'Names start with letters', 'supported', null, [], [], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
249        $specification = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
250        $results = (new Verifier())->verify($project, ['first' => $requirement, 'second' => $specification]);
251        self::assertSame(['REQ-1', 'SPEC-1'], array_keys($results));
252    }
253
254    public function testVerifyReturnsNothingWithoutItems(): void
255    {
256        $directory = new ProjectDirectory();
257        $project = new Project($directory->directory, [], [], [], [], [], 0.0, 0.0, [], []);
258        self::assertSame([], (new Verifier())->verify($project, []));
259    }
260
261    public function testVerifyRejectsAnUnknownRunnerExtension(): void
262    {
263        $directory = new ProjectDirectory();
264        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('jest', ['jest'], $directory->directory)], [], [], 0.0, 0.0, [], []);
265        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'a')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
266        $this->expectException(InvalidInputException::class);
267        $this->expectExceptionMessage('Unknown runner extension: jest');
268        (new Verifier())->verify($project, ['SPEC-1' => $item]);
269    }
270
271    public function testVerifyRejectsAnExtensionClassThatIsNotARunner(): void
272    {
273        $directory = new ProjectDirectory();
274        $project = new Project($directory->directory, [], [], [], [], ['custom' => stdClass::class], 0.0, 0.0, [], []);
275        $this->expectException(InvalidInputException::class);
276        $this->expectExceptionMessage('stdClass must implement RunnerExtension.');
277        (new Verifier())->verify($project, []);
278    }
279    #[DataProvider('providerExecutionPolicies')]
280    public function testVerifyDefersManualTargetsUnlessAllIsRequested(bool $all, bool $noTest, string $automatic, string $status, ?int $passed, int $deferred): void
281    {
282        $directory = new ProjectDirectory();
283        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
284        $item = new Item('SPEC-1', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', $automatic), new TestReference('example', 'slow', 'manual')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
285        $result = (new Verifier())->verify($project, ['SPEC-1' => $item], $noTest, $all)['SPEC-1'];
286        self::assertSame($status, $result->status);
287        self::assertSame($passed, $result->passedTargets);
288        self::assertSame($deferred, $result->deferredTargets);
289        self::assertSame(2, $result->totalTargets);
290        self::assertSame($noTest ? '' : $automatic . "\n" . ($all ? "slow\n" : ''), $directory->read('executions.txt'));
291    }
292
293    /**
294     * @return list<array{bool, bool, string, string, ?int, int}>
295     */
296    public static function providerExecutionPolicies(): array
297    {
298        return [
299            [false, false, 'fast', 'deferred', 1, 1],
300            [true, false, 'fast', 'passed', 2, 0],
301            [false, false, 'empty', 'failed', 0, 1],
302            [true, false, 'empty', 'failed', 1, 0],
303            [false, true, 'fast', 'not-run', null, 0],
304            [true, true, 'fast', 'not-run', null, 0],
305        ];
306    }
307
308    public function testVerifySharesAnAutomaticResultWithAnEarlierManualReference(): void
309    {
310        $directory = new ProjectDirectory();
311        $project = new Project($directory->directory, [], [], ['example' => new RunnerConfig('custom', ['example'], $directory->directory)], [], ['custom' => CountingRunner::class], 0.0, 0.0, [], []);
312        $manual = new Item('MANUAL', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'shared', 'manual')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
313        $automatic = new Item('AUTOMATIC', 'specification', 'The reader shall read.', 'supported', null, [], [new TestReference('example', 'shared')], [], [], [], '', 'original', 'Support editors.', 'definition.yaml', []);
314        $results = (new Verifier())->verify($project, ['MANUAL' => $manual, 'AUTOMATIC' => $automatic]);
315        self::assertSame('passed', $results['MANUAL']->status);
316        self::assertSame('passed', $results['AUTOMATIC']->status);
317        self::assertSame(0, $results['MANUAL']->deferredTargets);
318        self::assertSame("shared\n", $directory->read('executions.txt'));
319    }
320}
321