packages/requirements/tests/Unit/Report/CoverageTest.php
1<?php
2
3declare(strict_types=1);
4
5namespace Tests\Unit\Report;
6
7use JsonException;
8use PHPUnit\Framework\Attributes\CoversClass;
9use PHPUnit\Framework\Attributes\DataProvider;
10use PHPUnit\Framework\Attributes\Small;
11use PHPUnit\Framework\Attributes\UsesClass;
12use PHPUnit\Framework\TestCase;
13use Requirements\Config\CoverageThresholds;
14use Requirements\Config\DefinitionReader;
15use Requirements\Config\Definitions;
16use Requirements\Config\DocumentReader;
17use Requirements\Config\ExtensionClasses;
18use Requirements\Config\JsonSchemaFile;
19use Requirements\Config\LinkValidator;
20use Requirements\Config\Loader;
21use Requirements\Config\MarkdownDocument;
22use Requirements\Config\SchemaValidator;
23use Requirements\Ears\ConditionOrder;
24use Requirements\Ears\LiteralMask;
25use Requirements\Ears\SystemResponse;
26use Requirements\Ears\Validator;
27use Requirements\Ears\Wording;
28use Requirements\Input\Fields;
29use Requirements\Input\InvalidInputException;
30use Requirements\Model\Excerpt;
31use Requirements\Model\Item;
32use Requirements\Model\ItemValidator;
33use Requirements\Model\Project;
34use Requirements\Model\Source;
35use Requirements\Report\Analysis;
36use Requirements\Report\Analyzer;
37use Requirements\Report\Claims;
38use Requirements\Report\Coverage;
39use Requirements\Report\EvidenceMatcher;
40use Requirements\Report\Snapshot;
41use Requirements\Report\SourceUnit;
42use Requirements\Report\UnitCollector;
43use Requirements\Source\DomSource;
44use Requirements\Source\JsonSource;
45use Requirements\Source\LocalFile;
46use Requirements\Source\Registry;
47use Requirements\Source\ResourceLoader;
48use Requirements\Source\ResourceLocation;
49use Requirements\Source\TextFragment;
50use Requirements\Source\TextSource;
51use Requirements\Source\Unit;
52use Requirements\Test\Registry as RunnerRegistry;
53use Tests\Fake\ProjectDirectory;
54
55#[CoversClass(Coverage::class)]
56#[UsesClass(Analysis::class)]
57#[UsesClass(Analyzer::class)]
58#[UsesClass(Claims::class)]
59#[UsesClass(EvidenceMatcher::class)]
60#[UsesClass(Snapshot::class)]
61#[UsesClass(SourceUnit::class)]
62#[UsesClass(UnitCollector::class)]
63#[UsesClass(Loader::class)]
64#[UsesClass(Fields::class)]
65#[UsesClass(Item::class)]
66#[UsesClass(Project::class)]
67#[UsesClass(Source::class)]
68#[UsesClass(Unit::class)]
69#[UsesClass(CoverageThresholds::class)]
70#[UsesClass(DefinitionReader::class)]
71#[UsesClass(Definitions::class)]
72#[UsesClass(DocumentReader::class)]
73#[UsesClass(ExtensionClasses::class)]
74#[UsesClass(JsonSchemaFile::class)]
75#[UsesClass(LinkValidator::class)]
76#[UsesClass(MarkdownDocument::class)]
77#[UsesClass(SchemaValidator::class)]
78#[UsesClass(ConditionOrder::class)]
79#[UsesClass(LiteralMask::class)]
80#[UsesClass(SystemResponse::class)]
81#[UsesClass(Validator::class)]
82#[UsesClass(Wording::class)]
83#[UsesClass(Excerpt::class)]
84#[UsesClass(ItemValidator::class)]
85#[UsesClass(DomSource::class)]
86#[UsesClass(JsonSource::class)]
87#[UsesClass(LocalFile::class)]
88#[UsesClass(Registry::class)]
89#[UsesClass(ResourceLoader::class)]
90#[UsesClass(ResourceLocation::class)]
91#[UsesClass(TextFragment::class)]
92#[UsesClass(TextSource::class)]
93#[UsesClass(RunnerRegistry::class)]
94#[Small]
95final class CoverageTest extends TestCase
96{
97 /**
98 * @throws JsonException
99 */
100 public function testReportDescribesEveryUnitAndPassesWithoutGates(): void
101 {
102 $source = new Source('manual', 'source.html', 'html', 'main p');
103 $item = new Item('SPEC-001', 'specification', 'The parser shall read names.', 'supported', $source, [], [], [], [], [], '', 'sourced', '', 'definition.yaml', ['id' => 'SPEC-001']);
104 $claimed = new SourceUnit($source, new Unit('p:1', 'First.'));
105 $claimed->claims = ['SPEC-001' => $item];
106 $uncovered = new SourceUnit($source, new Unit('p:2', 'Second.'));
107 $project = new Project('/', ['SPEC-001' => $item], ['manual' => $source], [], [], [], 0.0, 0.0, [], []);
108 $analysis = new Analysis(['a' => $claimed, 'b' => $uncovered], ['manual' => ['a', 'b'], 'empty' => []], [], ['SPEC-001' => ['a']]);
109 self::assertSame([
110 'version' => 1,
111 'type' => 'requirements-coverage',
112 'overall' => ['total' => 2, 'accounted' => 1, 'supported' => 1, 'unsupported' => 0, 'uncovered' => 1, 'percentage' => 50.0],
113 'sources' => [
114 'manual' => ['total' => 2, 'accounted' => 1, 'supported' => 1, 'unsupported' => 0, 'uncovered' => 1, 'percentage' => 50.0],
115 'empty' => ['total' => 0, 'accounted' => 0, 'supported' => 0, 'unsupported' => 0, 'uncovered' => 0, 'percentage' => null],
116 ],
117 'diff' => null,
118 'removed' => [],
119 'units' => ['a' => $claimed->toArray(['SPEC-001' => $item]), 'b' => $uncovered->toArray([])],
120 'errors' => [],
121 'passed' => true,
122 ], (new Coverage())->report($project, $analysis));
123 }
124
125 /**
126 * @throws JsonException
127 */
128 public function testReportFailsWithTheErrorsOfTheAnalysis(): void
129 {
130 $project = new Project('/', [], [], [], [], [], 0.0, 0.0, [], []);
131 $report = (new Coverage())->report($project, new Analysis([], ['manual' => []], ['manual: Scope selected no source units.'], []));
132 self::assertSame(['manual: Scope selected no source units.'], $report['errors']);
133 self::assertFalse($report['passed']);
134 }
135
136 /**
137 * @param list<string> $errors
138 * @throws JsonException
139 */
140 #[DataProvider('providerOverallThresholds')]
141 public function testReportAppliesTheOverallThreshold(float $configured, ?float $override, array $errors): void
142 {
143 $source = new Source('manual', 'source.html', 'html', 'main p');
144 $item = new Item('SPEC-001', 'specification', 'The parser shall read names.', 'supported', $source, [], [], [], [], [], '', 'sourced', '', 'definition.yaml', []);
145 $claimed = new SourceUnit($source, new Unit('p:1', 'First.'));
146 $claimed->claims = ['SPEC-001' => $item];
147 $project = new Project('/', ['SPEC-001' => $item], ['manual' => $source], [], [], [], $configured, 0.0, [], []);
148 $analysis = new Analysis(['a' => $claimed, 'b' => new SourceUnit($source, new Unit('p:2', 'Second.'))], ['manual' => ['a', 'b']], [], []);
149 $report = (new Coverage())->report($project, $analysis, minimum: $override);
150 self::assertSame($errors, $report['errors']);
151 self::assertSame($errors === [], $report['passed']);
152 }
153
154 /**
155 * @return array<string, array{float, float|null, list<string>}>
156 */
157 public static function providerOverallThresholds(): array
158 {
159 return [
160 'configured below' => [40.0, null, []],
161 'configured equal' => [50.0, null, []],
162 'configured above' => [50.5, null, ['Overall source coverage is below 50.5%.']],
163 'override below configured' => [80.0, 50.0, []],
164 'override above configured' => [0.0, 80.0, ['Overall source coverage is below 80%.']],
165 ];
166 }
167
168 /**
169 * @throws JsonException
170 */
171 public function testReportFailsAPositiveOverallThresholdWithoutUnits(): void
172 {
173 $project = new Project('/', [], [], [], [], [], 0.0, 0.0, [], []);
174 $report = (new Coverage())->report($project, new Analysis([], [], [], []), minimum: 0.5);
175 self::assertSame(['Overall source coverage is below 0.5%.'], $report['errors']);
176 }
177
178 /**
179 * @param array<string, float> $thresholds
180 * @param list<string> $errors
181 * @throws JsonException
182 */
183 #[DataProvider('providerSourceThresholds')]
184 public function testReportAppliesTheSourceThresholds(array $thresholds, array $errors): void
185 {
186 $source = new Source('manual', 'source.html', 'html', 'main p');
187 $item = new Item('SPEC-001', 'specification', 'The parser shall read names.', 'supported', $source, [], [], [], [], [], '', 'sourced', '', 'definition.yaml', []);
188 $claimed = new SourceUnit($source, new Unit('p:1', 'First.'));
189 $claimed->claims = ['SPEC-001' => $item];
190 $project = new Project('/', ['SPEC-001' => $item], ['manual' => $source], [], [], [], 0.0, 0.0, $thresholds, []);
191 $analysis = new Analysis(['a' => $claimed, 'b' => new SourceUnit($source, new Unit('p:2', 'Second.'))], ['manual' => ['a', 'b'], 'full' => ['a'], 'empty' => []], [], []);
192 self::assertSame($errors, (new Coverage())->report($project, $analysis)['errors']);
193 }
194
195 /**
196 * @return array<string, array{array<string, float>, list<string>}>
197 */
198 public static function providerSourceThresholds(): array
199 {
200 return [
201 'none' => [[], []],
202 'equal' => [['manual' => 50.0], []],
203 'above' => [['manual' => 60.0, 'full' => 100.0], ['manual: source coverage is below 60%.']],
204 'empty scope' => [['empty' => 10.0], ['empty: source coverage is below 10%.']],
205 ];
206 }
207
208 /**
209 * @param list<string> $errors
210 * @throws JsonException
211 */
212 #[DataProvider('providerDifferentialThresholdsWithoutSnapshot')]
213 public function testReportRequiresASnapshotForAPositiveDifferentialThreshold(float $configured, ?float $override, array $errors): void
214 {
215 $project = new Project('/', [], [], [], [], [], 0.0, $configured, [], []);
216 $report = (new Coverage())->report($project, new Analysis([], [], [], []), diffMinimum: $override);
217 self::assertSame($errors, $report['errors']);
218 self::assertNull($report['diff']);
219 }
220
221 /**
222 * @return array<string, array{float, float|null, list<string>}>
223 */
224 public static function providerDifferentialThresholdsWithoutSnapshot(): array
225 {
226 return [
227 'zero' => [0.0, null, []],
228 'configured' => [0.5, null, ['A positive differential threshold requires --snapshot.']],
229 'override' => [0.0, 0.5, ['A positive differential threshold requires --snapshot.']],
230 'override to zero' => [50.0, 0.0, []],
231 ];
232 }
233
234 /**
235 * @param array{total: int, accounted: int, supported: int, unsupported: int, uncovered: int, percentage: float|null} $diff
236 * @param list<string> $errors
237 * @throws JsonException
238 */
239 #[DataProvider('providerDifferentialThresholds')]
240 public function testReportAppliesTheDifferentialThresholdToNewAndChangedUnits(bool $claimNew, float $configured, ?float $override, array $diff, array $errors): void
241 {
242 $workspace = new ProjectDirectory();
243 $source = new Source('manual', 'source.html', 'html', 'main p');
244 $item = new Item('SPEC-001', 'specification', 'The parser shall read names.', 'supported', $source, [], [], [], [], [], '', 'sourced', '', 'definition.yaml', []);
245 $old = new SourceUnit($source, new Unit('p:1', 'First.'));
246 $new = new SourceUnit($source, new Unit('p:2', 'Second.'));
247 $new->claims = $claimNew ? ['SPEC-001' => $item] : [];
248 $oldKey = $old->unit->key($source);
249 $newKey = $new->unit->key($source);
250 $file = $workspace->put('snapshot.json', json_encode(['version' => 1, 'type' => 'requirements-snapshot', 'units' => [$oldKey => ['fingerprint' => $old->fingerprint([])]]], JSON_THROW_ON_ERROR));
251 $project = new Project('/', ['SPEC-001' => $item], ['manual' => $source], [], [], [], 0.0, $configured, [], []);
252 $analysis = new Analysis([$oldKey => $old, $newKey => $new], ['manual' => [$oldKey, $newKey]], [], []);
253 $report = (new Coverage())->report($project, $analysis, $file, diffMinimum: $override);
254 self::assertSame($diff, $report['diff']);
255 self::assertSame([], $report['removed']);
256 self::assertSame($errors, $report['errors']);
257 }
258
259 /**
260 * @return array<string, array{bool, float, float|null, array{total: int, accounted: int, supported: int, unsupported: int, uncovered: int, percentage: float|null}, list<string>}>
261 */
262 public static function providerDifferentialThresholds(): array
263 {
264 $uncovered = ['total' => 1, 'accounted' => 0, 'supported' => 0, 'unsupported' => 0, 'uncovered' => 1, 'percentage' => 0.0];
265 $covered = ['total' => 1, 'accounted' => 1, 'supported' => 1, 'unsupported' => 0, 'uncovered' => 0, 'percentage' => 100.0];
266 return [
267 'no threshold' => [false, 0.0, null, $uncovered, []],
268 'configured' => [false, 50.0, null, $uncovered, ['Differential source coverage is below 50%.']],
269 'override' => [false, 0.0, 50.0, $uncovered, ['Differential source coverage is below 50%.']],
270 'override to zero' => [false, 50.0, 0.0, $uncovered, []],
271 'covered' => [true, 100.0, null, $covered, []],
272 ];
273 }
274
275 /**
276 * @throws JsonException
277 */
278 public function testReportSkipsTheDifferentialThresholdWithoutChangedUnits(): void
279 {
280 $workspace = new ProjectDirectory();
281 $source = new Source('manual', 'source.html', 'html', 'main p');
282 $unit = new SourceUnit($source, new Unit('p:1', 'First.'));
283 $key = $unit->unit->key($source);
284 $file = $workspace->put('snapshot.json', json_encode(['version' => 1, 'type' => 'requirements-snapshot', 'units' => [$key => ['fingerprint' => $unit->fingerprint([])]]], JSON_THROW_ON_ERROR));
285 $project = new Project('/', [], ['manual' => $source], [], [], [], 0.0, 100.0, [], []);
286 $report = (new Coverage())->report($project, new Analysis([$key => $unit], ['manual' => [$key]], [], []), $file);
287 self::assertSame(['total' => 0, 'accounted' => 0, 'supported' => 0, 'unsupported' => 0, 'uncovered' => 0, 'percentage' => null], $report['diff']);
288 self::assertSame([], $report['errors']);
289 self::assertTrue($report['passed']);
290 }
291
292 /**
293 * @param list<string> $errors
294 * @throws JsonException
295 */
296 #[DataProvider('providerRemovedUnits')]
297 public function testReportRejectsRemovedUnitsUnlessAllowed(bool $allowRemoved, array $errors): void
298 {
299 $workspace = new ProjectDirectory();
300 $source = new Source('manual', 'source.html', 'html', 'main p');
301 $unit = new SourceUnit($source, new Unit('p:1', 'First.'));
302 $key = $unit->unit->key($source);
303 $removed = hash('sha256', 'removed');
304 $file = $workspace->put('snapshot.json', json_encode(['version' => 1, 'type' => 'requirements-snapshot', 'units' => [$key => ['fingerprint' => $unit->fingerprint([])], $removed => ['fingerprint' => $removed]]], JSON_THROW_ON_ERROR));
305 $project = new Project('/', [], ['manual' => $source], [], [], [], 0.0, 0.0, [], []);
306 $report = (new Coverage())->report($project, new Analysis([$key => $unit], ['manual' => [$key]], [], []), $file, $allowRemoved);
307 self::assertSame([$removed], $report['removed']);
308 self::assertSame($errors, $report['errors']);
309 self::assertSame($errors === [], $report['passed']);
310 }
311
312 /**
313 * @return array<string, array{bool, list<string>}>
314 */
315 public static function providerRemovedUnits(): array
316 {
317 return [
318 'rejected' => [false, ['Source units listed in the snapshot were removed. Review scope changes before using --allow-removed.']],
319 'allowed' => [true, []],
320 ];
321 }
322
323 /**
324 * @throws JsonException
325 */
326 public function testReportRejectsAFileThatIsNotASnapshot(): void
327 {
328 $workspace = new ProjectDirectory();
329 $file = $workspace->put('coverage.json', '{"version":1,"type":"requirements-coverage"}');
330 $project = new Project('/', [], [], [], [], [], 0.0, 0.0, [], []);
331 $this->expectException(InvalidInputException::class);
332 $this->expectExceptionMessage('--write-snapshot');
333 (new Coverage())->report($project, new Analysis([], [], [], []), $file);
334 }
335
336 /**
337 * @throws JsonException
338 */
339 public function testReportFailsTheDifferentialGateForANewUncoveredUnit(): void
340 {
341 $workspace = new ProjectDirectory();
342 $loader = new Loader();
343 $project = $loader->load($workspace->directory . '/requirements.yaml');
344 $snapshot = $workspace->directory . '/snapshot.json';
345 file_put_contents($snapshot, json_encode((new Snapshot())->create((new Analyzer())->analyze($project), $project), JSON_THROW_ON_ERROR));
346 file_put_contents($workspace->directory . '/source.html', '<main><p id="a">Names shall start with a letter.</p><p id="b">Names may contain digits.</p><p id="c">The generator shall produce C code.</p><p id="d">New unreviewed rule.</p></main>');
347 $report = (new Coverage())->report($project, (new Analyzer())->analyze($project), $snapshot, diffMinimum: 100);
348 self::assertFalse($report['passed']);
349 self::assertIsArray($report['diff']);
350 self::assertSame(0.0, $report['diff']['percentage']);
351 }
352}
353