Skip to content

Commit 088e9cd

Browse files
panvaBethGriggs
panva
authored and
BethGriggs
committed
test: add WPTRunner support for variants and generating WPT reports
PR-URL: #46498 Backport-PR-URL: #46768 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Richard Lau <rlau@redhat.com>
1 parent 7af9bdb commit 088e9cd

File tree

2 files changed

+207
-57
lines changed

2 files changed

+207
-57
lines changed

‎Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,12 @@ test-message: test-build
573573
test-wpt: all
574574
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt
575575

576+
.PHONY: test-wpt-report
577+
test-wpt-report:
578+
$(RM) -r out/wpt
579+
mkdir -p out/wpt
580+
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
581+
576582
.PHONY: test-simple
577583
test-simple: | cctest # Depends on 'all'.
578584
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential

‎test/common/wpt.js

Lines changed: 201 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,119 @@ const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
88
const events = require('events');
9+
const os = require('os');
910
const { inspect } = require('util');
1011
const { Worker } = require('worker_threads');
1112

13+
function getBrowserProperties() {
14+
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
15+
const release = /^\d+\.\d+\.\d+$/.test(version);
16+
const browser = {
17+
browser_channel: release ? 'stable' : 'experimental',
18+
browser_version: version,
19+
};
20+
21+
return browser;
22+
}
23+
24+
/**
25+
* Return one of three expected values
26+
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
27+
*/
28+
function getOs() {
29+
switch (os.type()) {
30+
case 'Linux':
31+
return 'linux';
32+
case 'Darwin':
33+
return 'mac';
34+
case 'Windows_NT':
35+
return 'win';
36+
default:
37+
throw new Error('Unsupported os.type()');
38+
}
39+
}
40+
41+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
42+
function sanitizeUnpairedSurrogates(str) {
43+
return str.replace(
44+
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
45+
function(_, low, prefix, high) {
46+
let output = prefix || ''; // Prefix may be undefined
47+
const string = low || high; // Only one of these alternates can match
48+
for (let i = 0; i < string.length; i++) {
49+
output += codeUnitStr(string[i]);
50+
}
51+
return output;
52+
});
53+
}
54+
55+
function codeUnitStr(char) {
56+
return 'U+' + char.charCodeAt(0).toString(16);
57+
}
58+
59+
class WPTReport {
60+
constructor() {
61+
this.results = [];
62+
this.time_start = Date.now();
63+
}
64+
65+
addResult(name, status) {
66+
const result = {
67+
test: name,
68+
status,
69+
subtests: [],
70+
addSubtest(name, status, message) {
71+
const subtest = {
72+
status,
73+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
74+
name: sanitizeUnpairedSurrogates(name),
75+
};
76+
if (message) {
77+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
78+
subtest.message = sanitizeUnpairedSurrogates(message);
79+
}
80+
this.subtests.push(subtest);
81+
return subtest;
82+
},
83+
};
84+
this.results.push(result);
85+
return result;
86+
}
87+
88+
write() {
89+
this.time_end = Date.now();
90+
this.results = this.results.filter((result) => {
91+
return result.status === 'SKIP' || result.subtests.length !== 0;
92+
}).map((result) => {
93+
const url = new URL(result.test, 'http://wpt');
94+
url.pathname = url.pathname.replace(/\.js$/, '.html');
95+
result.test = url.href.slice(url.origin.length);
96+
return result;
97+
});
98+
99+
if (fs.existsSync('out/wpt/wptreport.json')) {
100+
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
101+
this.results = [...prev.results, ...this.results];
102+
this.time_start = prev.time_start;
103+
this.time_end = Math.max(this.time_end, prev.time_end);
104+
this.run_info = prev.run_info;
105+
} else {
106+
/**
107+
* Return required and some optional properties
108+
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
109+
*/
110+
this.run_info = {
111+
product: 'node.js',
112+
...getBrowserProperties(),
113+
revision: process.env.WPT_REVISION || 'unknown',
114+
os: getOs(),
115+
};
116+
}
117+
118+
fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
119+
}
120+
}
121+
12122
// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
13123
// TODO: get rid of this half-baked harness in favor of the one
14124
// pulled from WPT
@@ -313,6 +423,10 @@ class WPTRunner {
313423
this.unexpectedFailures = [];
314424

315425
this.scriptsModifier = null;
426+
427+
if (process.env.WPT_REPORT != null) {
428+
this.report = new WPTReport();
429+
}
316430
}
317431

318432
/**
@@ -339,18 +453,27 @@ class WPTRunner {
339453
this.scriptsModifier = modifier;
340454
}
341455

342-
get fullInitScript() {
343-
if (this.initScript === null && this.dummyGlobalThisScript === null) {
456+
fullInitScript(hasSubsetScript, locationSearchString) {
457+
let { initScript } = this;
458+
if (hasSubsetScript || locationSearchString) {
459+
initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`;
460+
}
461+
462+
if (locationSearchString) {
463+
initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`;
464+
}
465+
466+
if (initScript === null && this.dummyGlobalThisScript === null) {
344467
return null;
345468
}
346469

347-
if (this.initScript === null) {
470+
if (initScript === null) {
348471
return this.dummyGlobalThisScript;
349472
} else if (this.dummyGlobalThisScript === null) {
350-
return this.initScript;
473+
return initScript;
351474
}
352475

353-
return `${this.dummyGlobalThisScript}\n\n//===\n${this.initScript}`;
476+
return `${this.dummyGlobalThisScript}\n\n//===\n${initScript}`;
354477
}
355478

356479
/**
@@ -398,15 +521,20 @@ class WPTRunner {
398521
for (const spec of queue) {
399522
const testFileName = spec.filename;
400523
const content = spec.getContent();
401-
const meta = spec.title = this.getMeta(content);
524+
const meta = spec.meta = this.getMeta(content);
402525

403526
const absolutePath = spec.getAbsolutePath();
404527
const relativePath = spec.getRelativePath();
405528
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
406529
const scriptsToRun = [];
530+
let hasSubsetScript = false;
531+
407532
// Scripts specified with the `// META: script=` header
408533
if (meta.script) {
409534
for (const script of meta.script) {
535+
if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') {
536+
hasSubsetScript = true;
537+
}
410538
const obj = {
411539
filename: this.resource.toRealFilePath(relativePath, script),
412540
code: this.resource.read(relativePath, script, false)
@@ -423,54 +551,65 @@ class WPTRunner {
423551
this.scriptsModifier?.(obj);
424552
scriptsToRun.push(obj);
425553

426-
const workerPath = path.join(__dirname, 'wpt/worker.js');
427-
const worker = new Worker(workerPath, {
428-
execArgv: this.flags,
429-
workerData: {
430-
testRelativePath: relativePath,
431-
wptRunner: __filename,
432-
wptPath: this.path,
433-
initScript: this.fullInitScript,
434-
harness: {
435-
code: fs.readFileSync(harnessPath, 'utf8'),
436-
filename: harnessPath,
554+
/**
555+
* Example test with no META variant
556+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
557+
*
558+
* Example test with multiple META variants
559+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
560+
*/
561+
for (const variant of meta.variant || ['']) {
562+
const workerPath = path.join(__dirname, 'wpt/worker.js');
563+
const worker = new Worker(workerPath, {
564+
execArgv: this.flags,
565+
workerData: {
566+
testRelativePath: relativePath,
567+
wptRunner: __filename,
568+
wptPath: this.path,
569+
initScript: this.fullInitScript(hasSubsetScript, variant),
570+
harness: {
571+
code: fs.readFileSync(harnessPath, 'utf8'),
572+
filename: harnessPath,
573+
},
574+
scriptsToRun,
437575
},
438-
scriptsToRun,
439-
},
440-
});
441-
this.workers.set(testFileName, worker);
442-
443-
worker.on('message', (message) => {
444-
switch (message.type) {
445-
case 'result':
446-
return this.resultCallback(testFileName, message.result);
447-
case 'completion':
448-
return this.completionCallback(testFileName, message.status);
449-
default:
450-
throw new Error(`Unexpected message from worker: ${message.type}`);
451-
}
452-
});
576+
});
577+
this.workers.set(testFileName, worker);
578+
579+
let reportResult;
580+
worker.on('message', (message) => {
581+
switch (message.type) {
582+
case 'result':
583+
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
584+
return this.resultCallback(testFileName, message.result, reportResult);
585+
case 'completion':
586+
return this.completionCallback(testFileName, message.status);
587+
default:
588+
throw new Error(`Unexpected message from worker: ${message.type}`);
589+
}
590+
});
453591

454-
worker.on('error', (err) => {
455-
if (!this.inProgress.has(testFileName)) {
592+
worker.on('error', (err) => {
593+
if (!this.inProgress.has(testFileName)) {
456594
// The test is already finished. Ignore errors that occur after it.
457595
// This can happen normally, for example in timers tests.
458-
return;
459-
}
460-
this.fail(
461-
testFileName,
462-
{
463-
status: NODE_UNCAUGHT,
464-
name: 'evaluation in WPTRunner.runJsTests()',
465-
message: err.message,
466-
stack: inspect(err)
467-
},
468-
kUncaught
469-
);
470-
this.inProgress.delete(testFileName);
471-
});
596+
return;
597+
}
598+
this.fail(
599+
testFileName,
600+
{
601+
status: NODE_UNCAUGHT,
602+
name: 'evaluation in WPTRunner.runJsTests()',
603+
message: err.message,
604+
stack: inspect(err)
605+
},
606+
kUncaught
607+
);
608+
this.inProgress.delete(testFileName);
609+
});
472610

473-
await events.once(worker, 'exit').catch(() => {});
611+
await events.once(worker, 'exit').catch(() => {});
612+
}
474613
}
475614

476615
process.on('exit', () => {
@@ -529,6 +668,8 @@ class WPTRunner {
529668
}
530669
}
531670

671+
this.report?.write();
672+
532673
const ran = total - skipped;
533674
const passed = ran - expectedFailures - failures.length;
534675
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
@@ -552,8 +693,7 @@ class WPTRunner {
552693

553694
getTestTitle(filename) {
554695
const spec = this.specMap.get(filename);
555-
const title = spec.meta && spec.meta.title;
556-
return title ? `${filename} : ${title}` : filename;
696+
return spec.meta?.title || filename;
557697
}
558698

559699
// Map WPT test status to strings
@@ -579,14 +719,14 @@ class WPTRunner {
579719
* @param {string} filename
580720
* @param {Test} test The Test object returned by WPT harness
581721
*/
582-
resultCallback(filename, test) {
722+
resultCallback(filename, test, reportResult) {
583723
const status = this.getTestStatus(test.status);
584724
const title = this.getTestTitle(filename);
585725
console.log(`---- ${title} ----`);
586726
if (status !== kPass) {
587-
this.fail(filename, test, status);
727+
this.fail(filename, test, status, reportResult);
588728
} else {
589-
this.succeed(filename, test, status);
729+
this.succeed(filename, test, status, reportResult);
590730
}
591731
}
592732

@@ -634,11 +774,12 @@ class WPTRunner {
634774
}
635775
}
636776

637-
succeed(filename, test, status) {
777+
succeed(filename, test, status, reportResult) {
638778
console.log(`[${status.toUpperCase()}] ${test.name}`);
779+
reportResult?.addSubtest(test.name, 'PASS');
639780
}
640781

641-
fail(filename, test, status) {
782+
fail(filename, test, status, reportResult) {
642783
const spec = this.specMap.get(filename);
643784
const expected = spec.failedTests.includes(test.name);
644785
if (expected) {
@@ -654,6 +795,9 @@ class WPTRunner {
654795
const command = `${process.execPath} ${process.execArgv}` +
655796
` ${require.main.filename} ${filename}`;
656797
console.log(`Command: ${command}\n`);
798+
799+
reportResult?.addSubtest(test.name, 'FAIL', test.message);
800+
657801
this.addTestResult(filename, {
658802
name: test.name,
659803
expected,
@@ -683,7 +827,7 @@ class WPTRunner {
683827
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
684828
const key = parts[1];
685829
const value = parts[2];
686-
if (key === 'script') {
830+
if (key === 'script' || key === 'variant') {
687831
if (result[key]) {
688832
result[key].push(value);
689833
} else {

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy