packages/requirements/tests/Unit/Report/AnalyzerTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Report;
6
7use JsonException;
8use PHPUnit\Framework\Attributes\CoversClass;
9use PHPUnit\Framework\Attributes\DataProvider;
10use PHPUnit\Framework\Attributes\Small;
11use PHPUnit\Framework\Attributes\UsesClass;
12use PHPUnit\Framework\TestCase;
13use Requirements\Config\CoverageThresholds;
14use Requirements\Config\DefinitionReader;
15use Requirements\Config\Definitions;
16use Requirements\Config\DocumentReader;
17use Requirements\Config\ExtensionClasses;
18use Requirements\Config\JsonSchemaFile;
19use Requirements\Config\LinkValidator;
20use Requirements\Config\Loader;
21use Requirements\Config\MarkdownDocument;
22use Requirements\Config\SchemaValidator;
23use Requirements\Ears\ConditionOrder;
24use Requirements\Ears\LiteralMask;
25use Requirements\Ears\SystemResponse;
26use Requirements\Ears\Validator;
27use Requirements\Ears\Wording;
28use Requirements\Input\Fields;
29use Requirements\Model\Excerpt;
30use Requirements\Model\Item;
31use Requirements\Model\ItemValidator;
32use Requirements\Model\Project;
33use Requirements\Model\Source;
34use Requirements\Report\Analysis;
35use Requirements\Report\Analyzer;
36use Requirements\Report\Claims;
37use Requirements\Report\Coverage;
38use Requirements\Report\EvidenceMatcher;
39use Requirements\Report\SourceUnit;
40use Requirements\Report\UnitCollector;
41use Requirements\Source\DomSource;
42use Requirements\Source\JsonSource;
43use Requirements\Source\LocalFile;
44use Requirements\Source\Registry;
45use Requirements\Source\ResourceLoader;
46use Requirements\Source\ResourceLocation;
47use Requirements\Source\TextFragment;
48use Requirements\Source\TextSource;
49use Requirements\Source\Unit;
50use Requirements\Test\Registry as RunnerRegistry;
51use Tests\Fake\ProjectDirectory;
52use Tests\Fake\UnreadableSource;
53
54#[CoversClass(Analyzer::class)]
55#[UsesClass(Analysis::class)]
56#[UsesClass(Claims::class)]
57#[UsesClass(Coverage::class)]
58#[UsesClass(EvidenceMatcher::class)]
59#[UsesClass(SourceUnit::class)]
60#[UsesClass(UnitCollector::class)]
61#[UsesClass(Loader::class)]
62#[UsesClass(Fields::class)]
63#[UsesClass(Excerpt::class)]
64#[UsesClass(Item::class)]
65#[UsesClass(Project::class)]
66#[UsesClass(Source::class)]
67#[UsesClass(Registry::class)]
68#[UsesClass(Unit::class)]
69#[UsesClass(CoverageThresholds::class)]
70#[UsesClass(DefinitionReader::class)]
71#[UsesClass(Definitions::class)]
72#[UsesClass(DocumentReader::class)]
73#[UsesClass(ExtensionClasses::class)]
74#[UsesClass(JsonSchemaFile::class)]
75#[UsesClass(LinkValidator::class)]
76#[UsesClass(MarkdownDocument::class)]
77#[UsesClass(SchemaValidator::class)]
78#[UsesClass(ConditionOrder::class)]
79#[UsesClass(LiteralMask::class)]
80#[UsesClass(SystemResponse::class)]
81#[UsesClass(Validator::class)]
82#[UsesClass(Wording::class)]
83#[UsesClass(ItemValidator::class)]
84#[UsesClass(DomSource::class)]
85#[UsesClass(JsonSource::class)]
86#[UsesClass(LocalFile::class)]
87#[UsesClass(ResourceLoader::class)]
88#[UsesClass(ResourceLocation::class)]
89#[UsesClass(TextFragment::class)]
90#[UsesClass(TextSource::class)]
91#[UsesClass(RunnerRegistry::class)]
92#[Small]
93final class AnalyzerTest extends TestCase
94{
95 /**
96 * @throws JsonException
97 */
98 public function testAnalyzeUsesIndependentScopeAndReasonedDisposition(): void
99 {
100 $workspace = new ProjectDirectory();
101 $loader = new Loader();
102 $definition = $loader->document($workspace->directory . '/definition.yaml');
103 $unsupported = ProjectDirectory::item();
104 $unsupported['id'] = 'UNSUPPORTED-001';
105 $unsupported['status'] = 'unsupported';
106 $unsupported['reason'] = 'This library reads grammars; it does not generate C.';
107 $unsupported['statement'] = 'The generator shall produce C code.';
108 $unsupported['evidence'] = [['selector' => '#c', 'quote' => 'The generator shall produce C code.']];
109 $definition['items'] = [ProjectDirectory::item(), $unsupported];
110 $workspace->write('definition.yaml', $definition);
111 $project = $loader->load($workspace->directory . '/requirements.yaml');
112 $analysis = (new Analyzer())->analyze($project);
113 self::assertSame([], $analysis->errors);
114 self::assertSame(['total' => 3, 'accounted' => 2, 'supported' => 1, 'unsupported' => 1, 'uncovered' => 1, 'percentage' => 200.0 / 3], $analysis->summary());
115 self::assertFalse((new Coverage())->report($project, $analysis, minimum: 80)['passed']);
116 }
117
118 /**
119 * @throws JsonException
120 */
121 #[DataProvider('providerInvalidEvidence')]
122 public function testAnalyzeFailsClosedOnInvalidEvidence(string $selector, string $quote, string $error): void
123 {
124 $workspace = new ProjectDirectory();
125 $loader = new Loader();
126 $definition = $loader->document($workspace->directory . '/definition.yaml');
127 $item = ProjectDirectory::item();
128 $item['evidence'] = [['selector' => $selector, 'quote' => $quote]];
129 $definition['items'] = [$item];
130 $workspace->write('definition.yaml', $definition);
131 $analysis = (new Analyzer())->analyze($loader->load($workspace->directory . '/requirements.yaml'));
132 self::assertStringContainsString($error, implode(' ', $analysis->errors));
133 self::assertSame(0, $analysis->summary()['accounted']);
134 }
135
136 /**
137 * @return list<array{string, string, string}>
138 */
139 public static function providerInvalidEvidence(): array
140 {
141 return [
142 ['#missing', 'Nothing.', 'exactly one'],
143 ['main p', 'Names shall start with a letter.', 'exactly one'],
144 ['#a', 'start with a letter', 'differs'],
145 ['#outside', 'Outside scope.', 'outside'],
146 ];
147 }
148
149 /**
150 * @throws JsonException
151 */
152 public function testAnalyzeReportsAnEmptyScopeAsAnErrorAndNotAsFullCoverage(): void
153 {
154 $workspace = new ProjectDirectory();
155 $definition = (new Loader())->document($workspace->directory . '/definition.yaml');
156 $definition['source'] = ['id' => 'manual', 'uri' => 'source.html', 'format' => 'html', 'selector' => '#missing'];
157 $definition['items'] = [];
158 $workspace->write('definition.yaml', $definition);
159 $analysis = (new Analyzer())->analyze((new Loader())->load($workspace->directory . '/requirements.yaml'));
160 self::assertNotEmpty($analysis->errors);
161 self::assertNull($analysis->summary()['percentage']);
162 }
163
164 /**
165 * @throws JsonException
166 */
167 public function testAnalyzeNeedsASpecificationToClaimARequirementSharedAcrossDefinitions(): void
168 {
169 $workspace = new ProjectDirectory();
170 $loader = new Loader();
171 $definition = $loader->document($workspace->directory . '/definition.yaml');
172 $requirement = ProjectDirectory::item();
173 $requirement['kind'] = 'requirement';
174 $requirement['id'] = 'REQ-001';
175 $definition['items'] = [$requirement];
176 $workspace->write('definition.yaml', $definition);
177 self::assertSame(0, (new Analyzer())->analyze($loader->load($workspace->directory . '/requirements.yaml'))->summary()['accounted']);
178 $specification = ['id' => 'SPEC-001', 'statement' => 'The parser shall require a leading letter.', 'requirements' => ['REQ-001']];
179 $workspace->write('other.yaml', ['version' => 1, 'source' => null, 'items' => [$specification]]);
180 $workspace->write('requirements.yaml', ['version' => 1, 'definitions' => ['definition.yaml', 'other.yaml']]);
181 $analysis = (new Analyzer())->analyze($loader->load($workspace->directory . '/requirements.yaml'));
182 self::assertSame([], $analysis->errors);
183 self::assertSame(1, $analysis->summary()['accounted']);
184 }
185
186 /**
187 * @throws JsonException
188 */
189 public function testAnalyzeCountsUnitsSharedByScopesOnce(): void
190 {
191 $workspace = new ProjectDirectory();
192 $definition = (new Loader())->document($workspace->directory . '/definition.yaml');
193 $definition['source'] = ['id' => 'overlap', 'uri' => 'source.html', 'format' => 'html', 'selector' => '#a'];
194 $definition['items'] = [];
195 $workspace->write('other.yaml', $definition);
196 $workspace->write('requirements.yaml', ['version' => 1, 'definitions' => ['definition.yaml', 'other.yaml']]);
197 $analysis = (new Analyzer())->analyze((new Loader())->load($workspace->directory . '/requirements.yaml'));
198 self::assertSame(3, $analysis->summary()['total']);
199 self::assertSame(100.0, $analysis->summary($analysis->scopes['overlap'])['percentage']);
200 }
201
202 /**
203 * @throws JsonException
204 */
205 public function testAnalyzeRejectsEvidenceQuotingRepeatedUnitsAndCountsThemUncovered(): void
206 {
207 $workspace = new ProjectDirectory();
208 file_put_contents($workspace->directory . '/source.html', '<main><p>Repeated text.</p><p>Repeated text.</p></main>');
209 $workspace->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'manual', 'uri' => 'source.html', 'format' => 'html', 'selector' => 'main p'], 'items' => [['id' => 'SPEC-001', 'statement' => 'The reader shall preserve text.', 'evidence' => [['selector' => '#:~:text=Repeated%20text.', 'quote' => 'Repeated text.']]]]]);
210 $analysis = (new Analyzer())->analyze((new Loader())->load($workspace->directory . '/requirements.yaml'));
211 self::assertStringContainsString('exactly one unit', implode(' ', $analysis->errors));
212 self::assertSame(2, $analysis->summary()['total']);
213 self::assertSame(0, $analysis->summary()['accounted']);
214 }
215
216 /**
217 * @throws JsonException
218 */
219 public function testAnalyzeRecordsScopesEvidenceAndClaims(): void
220 {
221 $workspace = new ProjectDirectory();
222 $project = (new Loader())->load($workspace->directory . '/requirements.yaml');
223 $source = $project->sources['manual'];
224 $analysis = (new Analyzer())->analyze($project);
225 $keys = [(new Unit('/html/body/main/p[1]', ''))->key($source), (new Unit('/html/body/main/p[2]', ''))->key($source), (new Unit('/html/body/main/p[3]', ''))->key($source)];
226 self::assertSame([], $analysis->errors);
227 self::assertSame(['manual' => $keys], $analysis->scopes);
228 self::assertSame(['SPEC-001' => [$keys[0]]], $analysis->evidence);
229 self::assertSame(['SPEC-001'], array_keys($analysis->units[$keys[0]]->claims));
230 self::assertSame(['total' => 3, 'accounted' => 1, 'supported' => 1, 'unsupported' => 0, 'uncovered' => 2, 'percentage' => 100.0 / 3], $analysis->summary());
231 }
232
233 /**
234 * @throws JsonException
235 */
236 public function testAnalyzeReportsAFailingSourceExtensionForTheScopeAndEveryEvidenceEntry(): void
237 {
238 $workspace = new ProjectDirectory();
239 $workspace->write('requirements.yaml', ['version' => 1, 'definitions' => ['definition.yaml'], 'extensions' => ['sources' => ['unreadable' => UnreadableSource::class]]]);
240 $workspace->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'manual', 'uri' => 'source.html', 'format' => 'unreadable', 'selector' => 'main p'], 'items' => [ProjectDirectory::item()]]);
241 $analysis = (new Analyzer())->analyze((new Loader())->load($workspace->directory . '/requirements.yaml'));
242 self::assertSame(['manual: Cannot read source.html.', 'SPEC-001: Cannot read source.html.'], $analysis->errors);
243 self::assertSame([], $analysis->units);
244 self::assertSame(['manual' => []], $analysis->scopes);
245 self::assertSame(['SPEC-001' => []], $analysis->evidence);
246 }
247
248 /**
249 * @throws JsonException
250 */
251 public function testAnalyzeContinuesAfterAFailingScope(): void
252 {
253 $workspace = new ProjectDirectory();
254 $workspace->write('requirements.yaml', ['version' => 1, 'definitions' => ['broken.yaml', 'definition.yaml'], 'extensions' => ['sources' => ['unreadable' => UnreadableSource::class]]]);
255 $workspace->write('broken.yaml', ['version' => 1, 'source' => ['id' => 'broken', 'uri' => 'broken.html', 'format' => 'unreadable', 'selector' => 'p'], 'items' => []]);
256 $analysis = (new Analyzer())->analyze((new Loader())->load($workspace->directory . '/requirements.yaml'));
257 self::assertSame(['broken: Cannot read broken.html.'], $analysis->errors);
258 self::assertSame([], $analysis->scopes['broken']);
259 self::assertCount(3, $analysis->scopes['manual']);
260 self::assertSame(['total' => 3, 'accounted' => 1, 'supported' => 1, 'unsupported' => 0, 'uncovered' => 2, 'percentage' => 100.0 / 3], $analysis->summary());
261 }
262
263 /**
264 * @throws JsonException
265 */
266 public function testAnalyzeResolvesTheEvidenceOfItemsAfterAnItemWithoutSource(): void
267 {
268 $workspace = new ProjectDirectory();
269 $workspace->write('requirements.yaml', ['version' => 1, 'definitions' => ['charter.yaml', 'definition.yaml']]);
270 $workspace->write('charter.yaml', ['version' => 1, 'source' => null, 'items' => [['id' => 'SPEC-000', 'statement' => 'The parser shall report its version.', 'origin' => 'original', 'reason' => 'A project convention.']]]);
271 $project = (new Loader())->load($workspace->directory . '/requirements.yaml');
272 $analysis = (new Analyzer())->analyze($project);
273 self::assertSame(['SPEC-000', 'SPEC-001'], array_keys($project->items));
274 self::assertSame(['SPEC-000' => [], 'SPEC-001' => [(new Unit('/html/body/main/p[1]', ''))->key($project->sources['manual'])]], $analysis->evidence);
275 self::assertSame(1, $analysis->summary()['accounted']);
276 }
277
278 /**
279 * @throws JsonException
280 */
281 public function testAnalyzeKeepsTheValidEvidenceOfAnItemThatClaimsNothing(): void
282 {
283 $workspace = new ProjectDirectory();
284 $item = ProjectDirectory::item();
285 $item['evidence'] = [['selector' => '#missing', 'quote' => 'Nothing.'], ['selector' => '#b', 'quote' => 'Names may contain digits.'], ['selector' => '#outside', 'quote' => 'Outside scope.']];
286 $workspace->write('definition.yaml', ['version' => 1, 'source' => ['id' => 'manual', 'uri' => 'source.html', 'format' => 'html', 'selector' => 'main p'], 'items' => [$item]]);
287 $project = (new Loader())->load($workspace->directory . '/requirements.yaml');
288 $analysis = (new Analyzer())->analyze($project);
289 self::assertSame(['SPEC-001: Evidence must select exactly one unit: #missing', 'SPEC-001: Evidence is outside the declared scope: #outside'], $analysis->errors);
290 self::assertSame(['SPEC-001' => [(new Unit('/html/body/main/p[2]', ''))->key($project->sources['manual'])]], $analysis->evidence);
291 self::assertSame(0, $analysis->summary()['accounted']);
292 }
293
294 #[DataProvider('providerReadingModes')]
295 public function testAnalyzeReadsPinnedSnapshotsUnlessLive(bool $live, string $text): void
296 {
297 $workspace = new ProjectDirectory();
298 $workspace->put('current.txt', 'New text.');
299 $workspace->put('snapshot.txt', 'Old text.');
300 $source = new Source('notes', 'current.txt', 'text', 'lines:1', 'snapshot.txt', hash('sha256', 'Old text.'));
301 $item = new Item('SPEC-001', 'specification', 'The reader shall preserve text.', 'supported', $source, [new Excerpt('lines:1', $text)], [], [], [], [], '', 'sourced', '', 'definition.yaml', []);
302 $project = new Project($workspace->directory, ['SPEC-001' => $item], ['notes' => $source], [], [], [], 0.0, 0.0, [], []);
303 $analysis = (new Analyzer())->analyze($project, $live);
304 $key = (new Unit('line:1', $text))->key($source);
305 self::assertSame([], $analysis->errors);
306 self::assertSame($text, $analysis->units[$key]->unit->text);
307 self::assertSame(['SPEC-001' => [$key]], $analysis->evidence);
308 }
309
310 /**
311 * @return array<string, array{bool, string}>
312 */
313 public static function providerReadingModes(): array
314 {
315 return [
316 'snapshot' => [false, 'Old text.'],
317 'live' => [true, 'New text.'],
318 ];
319 }
320}
321