Skip to content

Commit d7c708a

Browse files
atlowChemitargos
atlowChemi
authored and
targos
committed
test_runner: add support for coverage via run()
PR-URL: #53937 Fixes: #53867 Refs: #53924 Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
1 parent 4244f1a commit d7c708a

File tree

3 files changed

+247
-1
lines changed

3 files changed

+247
-1
lines changed

‎doc/api/test.md

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,6 +1248,9 @@ added:
12481248
- v18.9.0
12491249
- v16.19.0
12501250
changes:
1251+
- version: REPLACEME
1252+
pr-url: https://github.com/nodejs/node/pull/53937
1253+
description: Added coverage options.
12511254
- version: v22.8.0
12521255
pr-url: https://github.com/nodejs/node/pull/53927
12531256
description: Added the `isolation` option.
@@ -1319,6 +1322,29 @@ changes:
13191322
that specifies the index of the shard to run. This option is _required_.
13201323
* `total` {number} is a positive integer that specifies the total number
13211324
of shards to split the test files to. This option is _required_.
1325+
* `coverage` {boolean} enable [code coverage][] collection.
1326+
**Default:** `false`.
1327+
* `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage
1328+
using a glob pattern, which can match both absolute and relative file paths.
1329+
This property is only applicable when `coverage` was set to `true`.
1330+
If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
1331+
files must meet **both** criteria to be included in the coverage report.
1332+
**Default:** `undefined`.
1333+
* `coverageIncludeGlobs` {string|Array} Includes specific files in code coverage
1334+
using a glob pattern, which can match both absolute and relative file paths.
1335+
This property is only applicable when `coverage` was set to `true`.
1336+
If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
1337+
files must meet **both** criteria to be included in the coverage report.
1338+
**Default:** `undefined`.
1339+
* `lineCoverage` {number} Require a minimum percent of covered lines. If code
1340+
coverage does not reach the threshold specified, the process will exit with code `1`.
1341+
**Default:** `0`.
1342+
* `branchCoverage` {number} Require a minimum percent of covered branches. If code
1343+
coverage does not reach the threshold specified, the process will exit with code `1`.
1344+
**Default:** `0`.
1345+
* `functionCoverage` {number} Require a minimum percent of covered functions. If code
1346+
coverage does not reach the threshold specified, the process will exit with code `1`.
1347+
**Default:** `0`.
13221348
* Returns: {TestsStream}
13231349

13241350
**Note:** `shard` is used to horizontally parallelize test running across
@@ -3527,6 +3553,7 @@ Can be used to abort test subtasks when the test has been aborted.
35273553
[`run()`]: #runoptions
35283554
[`suite()`]: #suitename-options-fn
35293555
[`test()`]: #testname-options-fn
3556+
[code coverage]: #collecting-code-coverage
35303557
[describe options]: #describename-options-fn
35313558
[it options]: #testname-options-fn
35323559
[stream.compose]: stream.md#streamcomposestreams

‎lib/internal/test_runner/runner.js

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ const {
5555
validateObject,
5656
validateOneOf,
5757
validateInteger,
58+
validateStringArray,
5859
} = require('internal/validators');
5960
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
6061
const { isRegExp } = require('internal/util/types');
@@ -524,7 +525,13 @@ function watchFiles(testFiles, opts) {
524525
function run(options = kEmptyObject) {
525526
validateObject(options, 'options');
526527

527-
let { testNamePatterns, testSkipPatterns, shard } = options;
528+
let {
529+
testNamePatterns,
530+
testSkipPatterns,
531+
shard,
532+
coverageExcludeGlobs,
533+
coverageIncludeGlobs,
534+
} = options;
528535
const {
529536
concurrency,
530537
timeout,
@@ -537,6 +544,10 @@ function run(options = kEmptyObject) {
537544
setup,
538545
only,
539546
globPatterns,
547+
coverage = false,
548+
lineCoverage = 0,
549+
branchCoverage = 0,
550+
functionCoverage = 0,
540551
} = options;
541552

542553
if (files != null) {
@@ -615,6 +626,22 @@ function run(options = kEmptyObject) {
615626
});
616627
}
617628
validateOneOf(isolation, 'options.isolation', ['process', 'none']);
629+
validateBoolean(coverage, 'options.coverage');
630+
if (coverageExcludeGlobs != null) {
631+
if (!ArrayIsArray(coverageExcludeGlobs)) {
632+
coverageExcludeGlobs = [coverageExcludeGlobs];
633+
}
634+
validateStringArray(coverageExcludeGlobs, 'options.coverageExcludeGlobs');
635+
}
636+
if (coverageIncludeGlobs != null) {
637+
if (!ArrayIsArray(coverageIncludeGlobs)) {
638+
coverageIncludeGlobs = [coverageIncludeGlobs];
639+
}
640+
validateStringArray(coverageIncludeGlobs, 'options.coverageIncludeGlobs');
641+
}
642+
validateInteger(lineCoverage, 'options.lineCoverage', 0, 100);
643+
validateInteger(branchCoverage, 'options.branchCoverage', 0, 100);
644+
validateInteger(functionCoverage, 'options.functionCoverage', 0, 100);
618645

619646
const rootTestOptions = { __proto__: null, concurrency, timeout, signal };
620647
const globalOptions = {
@@ -623,6 +650,12 @@ function run(options = kEmptyObject) {
623650
// behavior has relied on it, so removing it must be done in a semver major.
624651
...parseCommandLine(),
625652
setup, // This line can be removed when parseCommandLine() is removed here.
653+
coverage,
654+
coverageExcludeGlobs,
655+
coverageIncludeGlobs,
656+
lineCoverage: lineCoverage,
657+
branchCoverage: branchCoverage,
658+
functionCoverage: functionCoverage,
626659
};
627660
const root = createTestTree(rootTestOptions, globalOptions);
628661

‎test/parallel/test-runner-run-coverage.mjs

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
import * as common from '../common/index.mjs';
2+
import * as fixtures from '../common/fixtures.mjs';
3+
import { describe, it, run } from 'node:test';
4+
import assert from 'node:assert';
5+
import { sep } from 'node:path';
6+
7+
const files = [fixtures.path('test-runner', 'coverage.js')];
8+
const abortedSignal = AbortSignal.abort();
9+
10+
describe('require(\'node:test\').run coverage settings', { concurrency: true }, async () => {
11+
await describe('validation', async () => {
12+
await it('should only allow boolean in options.coverage', async () => {
13+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve(true), []]
14+
.forEach((coverage) => assert.throws(() => run({ coverage }), {
15+
code: 'ERR_INVALID_ARG_TYPE'
16+
}));
17+
});
18+
19+
await it('should only allow string|string[] in options.coverageExcludeGlobs', async () => {
20+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
21+
.forEach((coverageExcludeGlobs) => {
22+
assert.throws(() => run({ coverage: true, coverageExcludeGlobs }), {
23+
code: 'ERR_INVALID_ARG_TYPE'
24+
});
25+
assert.throws(() => run({ coverage: true, coverageExcludeGlobs: [coverageExcludeGlobs] }), {
26+
code: 'ERR_INVALID_ARG_TYPE'
27+
});
28+
});
29+
run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: [''] });
30+
run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: '' });
31+
});
32+
33+
await it('should only allow string|string[] in options.coverageIncludeGlobs', async () => {
34+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
35+
.forEach((coverageIncludeGlobs) => {
36+
assert.throws(() => run({ coverage: true, coverageIncludeGlobs }), {
37+
code: 'ERR_INVALID_ARG_TYPE'
38+
});
39+
assert.throws(() => run({ coverage: true, coverageIncludeGlobs: [coverageIncludeGlobs] }), {
40+
code: 'ERR_INVALID_ARG_TYPE'
41+
});
42+
});
43+
44+
run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: [''] });
45+
run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: '' });
46+
});
47+
48+
await it('should only allow an int within range in options.lineCoverage', async () => {
49+
[Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
50+
.forEach((lineCoverage) => {
51+
assert.throws(() => run({ coverage: true, lineCoverage }), {
52+
code: 'ERR_INVALID_ARG_TYPE'
53+
});
54+
assert.throws(() => run({ coverage: true, lineCoverage: [lineCoverage] }), {
55+
code: 'ERR_INVALID_ARG_TYPE'
56+
});
57+
});
58+
assert.throws(() => run({ coverage: true, lineCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
59+
assert.throws(() => run({ coverage: true, lineCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
60+
61+
run({ files: [], signal: abortedSignal, coverage: true, lineCoverage: 0 });
62+
});
63+
64+
await it('should only allow an int within range in options.branchCoverage', async () => {
65+
[Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
66+
.forEach((branchCoverage) => {
67+
assert.throws(() => run({ coverage: true, branchCoverage }), {
68+
code: 'ERR_INVALID_ARG_TYPE'
69+
});
70+
assert.throws(() => run({ coverage: true, branchCoverage: [branchCoverage] }), {
71+
code: 'ERR_INVALID_ARG_TYPE'
72+
});
73+
});
74+
75+
assert.throws(() => run({ coverage: true, branchCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
76+
assert.throws(() => run({ coverage: true, branchCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
77+
78+
run({ files: [], signal: abortedSignal, coverage: true, branchCoverage: 0 });
79+
});
80+
81+
await it('should only allow an int within range in options.functionCoverage', async () => {
82+
[Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
83+
.forEach((functionCoverage) => {
84+
assert.throws(() => run({ coverage: true, functionCoverage }), {
85+
code: 'ERR_INVALID_ARG_TYPE'
86+
});
87+
assert.throws(() => run({ coverage: true, functionCoverage: [functionCoverage] }), {
88+
code: 'ERR_INVALID_ARG_TYPE'
89+
});
90+
});
91+
92+
assert.throws(() => run({ coverage: true, functionCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
93+
assert.throws(() => run({ coverage: true, functionCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
94+
95+
run({ files: [], signal: abortedSignal, coverage: true, functionCoverage: 0 });
96+
});
97+
});
98+
99+
const options = { concurrency: false, skip: !process.features.inspector ? 'inspector disabled' : false };
100+
await describe('run with coverage', options, async () => {
101+
await it('should run with coverage', async () => {
102+
const stream = run({ files, coverage: true });
103+
stream.on('test:fail', common.mustNotCall());
104+
stream.on('test:pass', common.mustCall());
105+
stream.on('test:coverage', common.mustCall());
106+
// eslint-disable-next-line no-unused-vars
107+
for await (const _ of stream);
108+
});
109+
110+
await it('should run with coverage and exclude by glob', async () => {
111+
const stream = run({ files, coverage: true, coverageExcludeGlobs: ['test/*/test-runner/invalid-tap.js'] });
112+
stream.on('test:fail', common.mustNotCall());
113+
stream.on('test:pass', common.mustCall(1));
114+
stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
115+
const filesPaths = files.map(({ path }) => path);
116+
assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}invalid-tap.js`)), false);
117+
}));
118+
// eslint-disable-next-line no-unused-vars
119+
for await (const _ of stream);
120+
});
121+
122+
await it('should run with coverage and include by glob', async () => {
123+
const stream = run({
124+
files,
125+
coverage: true,
126+
coverageIncludeGlobs: ['test/fixtures/test-runner/coverage.js', 'test/*/v8-coverage/throw.js'],
127+
});
128+
stream.on('test:fail', common.mustNotCall());
129+
stream.on('test:pass', common.mustCall(1));
130+
stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
131+
const filesPaths = files.map(({ path }) => path);
132+
assert.strictEqual(filesPaths.some((path) => path.includes(`v8-coverage${sep}throw.js`)), true);
133+
}));
134+
// eslint-disable-next-line no-unused-vars
135+
for await (const _ of stream);
136+
});
137+
138+
await it('should run while including and excluding globs', async () => {
139+
const stream = run({
140+
files: [...files, fixtures.path('test-runner/invalid-tap.js')],
141+
coverage: true,
142+
coverageIncludeGlobs: ['test/fixtures/test-runner/*.js'],
143+
coverageExcludeGlobs: ['test/fixtures/test-runner/*-tap.js']
144+
});
145+
stream.on('test:fail', common.mustNotCall());
146+
stream.on('test:pass', common.mustCall(2));
147+
stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
148+
const filesPaths = files.map(({ path }) => path);
149+
assert.strictEqual(filesPaths.every((path) => !path.includes(`test-runner${sep}invalid-tap.js`)), true);
150+
assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}coverage.js`)), true);
151+
}));
152+
// eslint-disable-next-line no-unused-vars
153+
for await (const _ of stream);
154+
});
155+
156+
await it('should run with coverage and fail when below line threshold', async () => {
157+
const thresholdErrors = [];
158+
const originalExitCode = process.exitCode;
159+
assert.notStrictEqual(originalExitCode, 1);
160+
const stream = run({ files, coverage: true, lineCoverage: 99, branchCoverage: 99, functionCoverage: 99 });
161+
stream.on('test:fail', common.mustNotCall());
162+
stream.on('test:pass', common.mustCall(1));
163+
stream.on('test:diagnostic', ({ message }) => {
164+
const match = message.match(/Error: \d{2}\.\d{2}% (line|branch|function) coverage does not meet threshold of 99%/);
165+
if (match) {
166+
thresholdErrors.push(match[1]);
167+
}
168+
});
169+
// eslint-disable-next-line no-unused-vars
170+
for await (const _ of stream);
171+
assert.deepStrictEqual(thresholdErrors.sort(), ['branch', 'function', 'line']);
172+
assert.strictEqual(process.exitCode, 1);
173+
process.exitCode = originalExitCode;
174+
});
175+
});
176+
});
177+
178+
179+
// exitHandler doesn't run until after the tests / after hooks finish.
180+
process.on('exit', () => {
181+
assert.strictEqual(process.listeners('uncaughtException').length, 0);
182+
assert.strictEqual(process.listeners('unhandledRejection').length, 0);
183+
assert.strictEqual(process.listeners('beforeExit').length, 0);
184+
assert.strictEqual(process.listeners('SIGINT').length, 0);
185+
assert.strictEqual(process.listeners('SIGTERM').length, 0);
186+
});

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy