packages/requirements/tests/Unit/Console/ExecutorTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Console;
6
7use JsonException;
8use PHPUnit\Framework\Attributes\CoversClass;
9use PHPUnit\Framework\Attributes\DataProvider;
10use PHPUnit\Framework\Attributes\Small;
11use PHPUnit\Framework\Attributes\UsesClass;
12use PHPUnit\Framework\TestCase;
13use Requirements\Config\CoverageThresholds;
14use Requirements\Config\DefinitionReader;
15use Requirements\Config\Definitions;
16use Requirements\Config\DocumentReader;
17use Requirements\Config\ExtensionClasses;
18use Requirements\Config\JsonSchemaFile;
19use Requirements\Config\LinkValidator;
20use Requirements\Config\Loader;
21use Requirements\Config\MarkdownDocument;
22use Requirements\Config\SchemaValidator;
23use Requirements\Console\Executor;
24use Requirements\Console\Formatter;
25use Requirements\Console\ItemRecord;
26use Requirements\Console\Options;
27use Requirements\Console\SpecificationReport;
28use Requirements\Ears\ConditionOrder;
29use Requirements\Ears\LiteralMask;
30use Requirements\Ears\SystemResponse;
31use Requirements\Ears\Validator;
32use Requirements\Ears\Wording;
33use Requirements\Input\Fields;
34use Requirements\Input\InvalidInputException;
35use Requirements\Model\Excerpt;
36use Requirements\Model\Item;
37use Requirements\Model\ItemValidator;
38use Requirements\Model\Project;
39use Requirements\Model\Source;
40use Requirements\Model\TestReference;
41use Requirements\Report\Analysis;
42use Requirements\Report\Analyzer;
43use Requirements\Report\Claims;
44use Requirements\Report\Coverage;
45use Requirements\Report\EvidenceMatcher;
46use Requirements\Report\Snapshot;
47use Requirements\Report\SourceUnit;
48use Requirements\Report\UnitCollector;
49use Requirements\Source\DomSource;
50use Requirements\Source\JsonSource;
51use Requirements\Source\LocalFile;
52use Requirements\Source\Registry as SourceRegistry;
53use Requirements\Source\ResourceLoader;
54use Requirements\Source\ResourceLocation;
55use Requirements\Source\TextFragment;
56use Requirements\Source\TextSource;
57use Requirements\Source\Unit;
58use Requirements\Test\Registry as TestRegistry;
59use Requirements\Test\RunnerConfig;
60use Requirements\Test\TestResult;
61use Requirements\Verification\TargetResults;
62use Requirements\Verification\TestExecution;
63use Requirements\Verification\VerificationResult;
64use Requirements\Verification\Verifier;
65use Tests\Fake\CountingRunner;
66use Tests\Fake\ProjectDirectory;
67
68#[CoversClass(Executor::class)]
69#[UsesClass(SpecificationReport::class)]
70#[UsesClass(CoverageThresholds::class)]
71#[UsesClass(DefinitionReader::class)]
72#[UsesClass(Definitions::class)]
73#[UsesClass(DocumentReader::class)]
74#[UsesClass(ExtensionClasses::class)]
75#[UsesClass(JsonSchemaFile::class)]
76#[UsesClass(LinkValidator::class)]
77#[UsesClass(Loader::class)]
78#[UsesClass(MarkdownDocument::class)]
79#[UsesClass(SchemaValidator::class)]
80#[UsesClass(Formatter::class)]
81#[UsesClass(ItemRecord::class)]
82#[UsesClass(Options::class)]
83#[UsesClass(ConditionOrder::class)]
84#[UsesClass(LiteralMask::class)]
85#[UsesClass(SystemResponse::class)]
86#[UsesClass(Validator::class)]
87#[UsesClass(Wording::class)]
88#[UsesClass(Fields::class)]
89#[UsesClass(Excerpt::class)]
90#[UsesClass(Item::class)]
91#[UsesClass(ItemValidator::class)]
92#[UsesClass(Project::class)]
93#[UsesClass(Source::class)]
94#[UsesClass(TestReference::class)]
95#[UsesClass(Analysis::class)]
96#[UsesClass(Analyzer::class)]
97#[UsesClass(Claims::class)]
98#[UsesClass(Coverage::class)]
99#[UsesClass(EvidenceMatcher::class)]
100#[UsesClass(Snapshot::class)]
101#[UsesClass(SourceUnit::class)]
102#[UsesClass(UnitCollector::class)]
103#[UsesClass(DomSource::class)]
104#[UsesClass(JsonSource::class)]
105#[UsesClass(LocalFile::class)]
106#[UsesClass(SourceRegistry::class)]
107#[UsesClass(ResourceLoader::class)]
108#[UsesClass(ResourceLocation::class)]
109#[UsesClass(TextFragment::class)]
110#[UsesClass(TextSource::class)]
111#[UsesClass(Unit::class)]
112#[UsesClass(TestRegistry::class)]
113#[UsesClass(RunnerConfig::class)]
114#[UsesClass(TestResult::class)]
115#[UsesClass(VerificationResult::class)]
116#[UsesClass(Verifier::class)]
117#[UsesClass(TestExecution::class)]
118#[UsesClass(TargetResults::class)]
119#[Small]
120final class ExecutorTest extends TestCase
121{
122 /**
123 * @throws JsonException
124 */
125 public function testExecuteLintCountsTheValidatedItems(): void
126 {
127 $project = new ProjectDirectory();
128 $loaded = (new Loader())->load($project->path('requirements.yaml'));
129 self::assertSame(['passed' => true, 'message' => '1 items validated.'], (new Executor())->execute($loaded, new Options('lint', [])));
130 }
131
132 /**
133 * @throws JsonException
134 */
135 public function testExecuteFormatReportsNoChangesForCanonicalDocuments(): void
136 {
137 $project = new ProjectDirectory();
138 $loaded = (new Loader())->load($project->path('requirements.yaml'));
139 self::assertSame(['passed' => true, 'changed' => []], (new Executor())->execute($loaded, new Options('format', ['check' => true])));
140 }
141
142 /**
143 * @throws JsonException
144 */
145 public function testExecuteFormatFailsTheCheckWhenADocumentNeedsFormatting(): void
146 {
147 $project = new ProjectDirectory();
148 $project->put('requirements.yaml', "{version: 1, definitions: [definition.yaml]}\n");
149 $loaded = (new Loader())->load($project->path('requirements.yaml'));
150 self::assertSame(['passed' => false, 'changed' => [$project->path('requirements.yaml')]], (new Executor())->execute($loaded, new Options('format', ['check' => true])));
151 self::assertSame("{version: 1, definitions: [definition.yaml]}\n", $project->read('requirements.yaml'));
152 }
153
154 /**
155 * @throws JsonException
156 */
157 public function testExecuteFormatRewritesDocumentsAndPasses(): void
158 {
159 $project = new ProjectDirectory();
160 $project->put('requirements.yaml', "{version: 1, definitions: [definition.yaml]}\n");
161 $loaded = (new Loader())->load($project->path('requirements.yaml'));
162 self::assertSame(['passed' => true, 'changed' => [$project->path('requirements.yaml')]], (new Executor())->execute($loaded, new Options('format', [])));
163 self::assertSame("version: 1\ndefinitions:\n - definition.yaml\n", $project->read('requirements.yaml'));
164 }
165
166 /**
167 * @throws JsonException
168 */
169 public function testExecuteCheckReportsValidEvidence(): void
170 {
171 $project = new ProjectDirectory();
172 $loaded = (new Loader())->load($project->path('requirements.yaml'));
173 $report = (new Executor())->execute($loaded, new Options('check', []));
174 self::assertSame(['passed', 'mode', 'errors', 'evidence'], array_keys($report));
175 self::assertTrue($report['passed']);
176 self::assertSame('configured', $report['mode']);
177 self::assertSame([], $report['errors']);
178 self::assertSame(['SPEC-001'], array_keys(Fields::mapping($report['evidence'], 'evidence')));
179 }
180
181 /**
182 * @throws JsonException
183 */
184 public function testExecuteCheckReadsLiveSources(): void
185 {
186 $project = new ProjectDirectory();
187 $loaded = (new Loader())->load($project->path('requirements.yaml'));
188 $report = (new Executor())->execute($loaded, new Options('check', ['live' => true]));
189 self::assertTrue($report['passed']);
190 self::assertSame('live', $report['mode']);
191 }
192
193 /**
194 * @throws JsonException
195 */
196 public function testExecuteCheckFailsOnInvalidEvidence(): void
197 {
198 $project = new ProjectDirectory();
199 $project->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'manual', 'uri' => 'source.html', 'format' => 'html', 'selector' => 'main p'], 'items' => [[...ProjectDirectory::item(), 'evidence' => [['selector' => '#a', 'quote' => 'Names shall end with a letter.']]]]]);
200 $loaded = (new Loader())->load($project->path('requirements.yaml'));
201 $report = (new Executor())->execute($loaded, new Options('check', []));
202 self::assertFalse($report['passed']);
203 self::assertNotSame([], $report['errors']);
204 }
205
206 /**
207 * @throws JsonException
208 */
209 public function testExecuteCoverageReportsTheConfiguredMode(): void
210 {
211 $project = new ProjectDirectory();
212 $loaded = (new Loader())->load($project->path('requirements.yaml'));
213 $report = (new Executor())->execute($loaded, new Options('coverage', []));
214 self::assertTrue($report['passed']);
215 self::assertSame('configured', $report['mode']);
216 self::assertSame('requirements-coverage', $report['type']);
217 self::assertNull($report['diff']);
218 self::assertSame(3, Fields::mapping($report['overall'], 'overall')['total']);
219 }
220
221 /**
222 * @throws JsonException
223 */
224 public function testExecuteCoverageReportsTheLiveMode(): void
225 {
226 $project = new ProjectDirectory();
227 $loaded = (new Loader())->load($project->path('requirements.yaml'));
228 self::assertSame('live', (new Executor())->execute($loaded, new Options('coverage', ['live' => true]))['mode']);
229 }
230
231 /**
232 * @throws JsonException
233 */
234 public function testExecuteCoverageAppliesTheThresholdOptions(): void
235 {
236 $project = new ProjectDirectory();
237 $loaded = (new Loader())->load($project->path('requirements.yaml'));
238 $report = (new Executor())->execute($loaded, new Options('coverage', ['min-coverage' => '100', 'min-diff-coverage' => '50']));
239 self::assertFalse($report['passed']);
240 self::assertSame(['Overall source coverage is below 100%.', 'A positive differential threshold requires --snapshot.'], $report['errors']);
241 }
242
243 /**
244 * @throws JsonException
245 */
246 public function testExecuteCoverageWritesAndComparesASnapshot(): void
247 {
248 $project = new ProjectDirectory();
249 $loaded = (new Loader())->load($project->path('requirements.yaml'));
250 $written = (new Executor())->execute($loaded, new Options('coverage', ['write-snapshot' => $project->path('snapshot.json')]));
251 self::assertTrue($written['passed']);
252 $snapshot = Fields::mapping(json_decode($project->read('snapshot.json'), true, 512, JSON_THROW_ON_ERROR), 'snapshot');
253 self::assertSame(['version', 'type', 'units'], array_keys($snapshot));
254 self::assertSame('requirements-snapshot', $snapshot['type']);
255 self::assertStringEndsWith("}\n", $project->read('snapshot.json'));
256 self::assertStringContainsString("\n \"type\": \"requirements-snapshot\",\n", $project->read('snapshot.json'));
257 $compared = (new Executor())->execute($loaded, new Options('coverage', ['snapshot' => $project->path('snapshot.json'), 'min-diff-coverage' => '100']));
258 self::assertTrue($compared['passed']);
259 self::assertSame(['total' => 0, 'accounted' => 0, 'supported' => 0, 'unsupported' => 0, 'uncovered' => 0, 'percentage' => null], $compared['diff']);
260 self::assertSame([], $compared['removed']);
261 }
262
263 /**
264 * @throws JsonException
265 */
266 public function testExecuteCoverageRefusesASnapshotOfInvalidEvidence(): void
267 {
268 $project = new ProjectDirectory();
269 $project->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'manual', 'uri' => 'source.html', 'format' => 'html', 'selector' => 'main p'], 'items' => [[...ProjectDirectory::item(), 'evidence' => [['selector' => '#a', 'quote' => 'Names shall end with a letter.']]]]]);
270 $loaded = (new Loader())->load($project->path('requirements.yaml'));
271 $this->expectException(InvalidInputException::class);
272 $this->expectExceptionMessage('Cannot write a coverage snapshot with invalid source evidence.');
273 (new Executor())->execute($loaded, new Options('coverage', ['write-snapshot' => $project->path('snapshot.json')]));
274 }
275
276 /**
277 * @throws JsonException
278 */
279 public function testExecuteSpecListsRecordsWithoutRunningTests(): void
280 {
281 $project = new ProjectDirectory();
282 $loaded = (new Loader())->load($project->path('requirements.yaml'));
283 self::assertSame(['passed' => true, 'no_test' => true, 'strict' => false, 'all' => false, 'specifications' => ['SPEC-001' => [
284 'id' => 'SPEC-001',
285 'kind' => 'specification',
286 'statement' => 'When a name is read, the parser shall require a leading letter.',
287 'support' => 'supported',
288 'source' => 'manual',
289 'origin' => 'sourced',
290 'reason' => '',
291 'labels' => [],
292 'category' => '',
293 'requirements' => [],
294 'related' => [],
295 'design' => [],
296 'metadata' => [],
297 'test_references' => [],
298 'status' => 'not-run',
299 'tests' => 0,
300 'passed_targets' => null,
301 'total_targets' => 0,
302 'message' => '',
303 'deferred_targets' => 0,
304 ]], 'errors' => []], (new Executor())->execute($loaded, new Options('spec', ['no-test' => true])));
305 }
306
307 /**
308 * @throws JsonException
309 */
310 public function testExecuteSpecAcceptsEmptySelectionByDefault(): void
311 {
312 $project = new ProjectDirectory();
313 $loaded = (new Loader())->load($project->path('requirements.yaml'));
314 self::assertSame(['passed' => true, 'no_test' => true, 'strict' => false, 'all' => false, 'specifications' => [], 'errors' => []], (new Executor())->execute($loaded, new Options('spec', ['no-test' => true, 'id' => 'MISSING'])));
315 }
316
317 /**
318 * @param list<array<string, mixed>> $items
319 * @param array<string, string> $statuses
320 * @throws JsonException
321 */
322 #[DataProvider('providerExecuteSpec')]
323 public function testExecuteSpecPassesOnlyWhenEveryItemPasses(array $items, array $statuses, bool $passed): void
324 {
325 $project = new ProjectDirectory();
326 $project->write('requirements.yaml', [
327 'version' => 1,
328 'definitions' => ['definition.yaml'],
329 'extensions' => ['runners' => ['custom' => CountingRunner::class]],
330 'runners' => ['example' => ['extension' => 'custom', 'command' => ['example']]],
331 ]);
332 $project->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'manual', 'uri' => 'source.html', 'format' => 'html', 'selector' => 'main p'], 'items' => $items]);
333 $loaded = (new Loader())->load($project->path('requirements.yaml'));
334 $report = (new Executor())->execute($loaded, new Options('spec', []));
335 self::assertSame($passed, $report['passed']);
336 self::assertFalse($report['no_test']);
337 self::assertSame([], $report['errors']);
338 self::assertSame($statuses, array_column(Fields::mapping($report['specifications'], 'specifications'), 'status', 'id'));
339 }
340
341 /**
342 * @return array<string, array{list<array<string, mixed>>, array<string, string>, bool}>
343 */
344 public static function providerExecuteSpec(): array
345 {
346 $passing = [...ProjectDirectory::item(), 'id' => 'PASSING', 'tests' => [['runner' => 'example', 'target' => 'shared']]];
347 $failing = [...ProjectDirectory::item(), 'id' => 'FAILING', 'tests' => [['runner' => 'example', 'target' => 'empty']]];
348 $unverified = [...ProjectDirectory::item(), 'id' => 'UNVERIFIED'];
349 $unsupported = [...ProjectDirectory::item(), 'id' => 'UNSUPPORTED', 'status' => 'unsupported', 'reason' => 'The external system owns this behavior.'];
350 $requirement = ['id' => 'REQ-001', 'kind' => 'requirement', 'statement' => 'Names may contain digits.', 'evidence' => [['selector' => '#b', 'quote' => 'Names may contain digits.']]];
351 return [
352 'passing' => [[$passing], ['PASSING' => 'passed'], true],
353 'passing, unsupported and not applicable' => [[$passing, $unsupported, $requirement], ['PASSING' => 'passed', 'UNSUPPORTED' => 'unsupported', 'REQ-001' => 'not-applicable'], true],
354 'failing first' => [[$failing, $passing], ['FAILING' => 'failed', 'PASSING' => 'passed'], false],
355 'failing last' => [[$passing, $failing], ['PASSING' => 'passed', 'FAILING' => 'failed'], false],
356 'unverified' => [[$unverified, $passing], ['UNVERIFIED' => 'unverified', 'PASSING' => 'passed'], true],
357 ];
358 }
359
360 /**
361 * @throws JsonException
362 */
363 public function testExecuteRejectsAnUnknownCommand(): void
364 {
365 $project = new ProjectDirectory();
366 $loaded = (new Loader())->load($project->path('requirements.yaml'));
367 $this->expectException(InvalidInputException::class);
368 $this->expectExceptionMessage('Unknown command: unknown');
369 (new Executor())->execute($loaded, new Options('unknown', []));
370 }
371 /**
372 * @throws JsonException
373 */
374 public function testStrictSpecRequiresLinksEvenWithoutTestExecution(): void
375 {
376 $project = new ProjectDirectory();
377 $loaded = (new Loader())->load($project->path('requirements.yaml'));
378 $report = (new Executor())->execute($loaded, new Options('spec', ['no-test' => true, 'strict' => true]));
379 self::assertFalse($report['passed']);
380 self::assertSame(['SPEC-001: No tests linked.'], $report['errors']);
381 $empty = (new Executor())->execute($loaded, new Options('spec', ['id' => 'MISSING', 'strict' => true]));
382 self::assertFalse($empty['passed']);
383 self::assertSame(['No specifications or requirements selected.'], $empty['errors']);
384 }
385}
386