test: allow skipping individual WPT subtests

Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: #62517
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

Author: panva

test/common/wpt.js

@@ -224,6 +224,7 @@ class StatusRule {
     this.requires = value.requires || [];
     this.fail = value.fail;
     this.skip = value.skip;
+    this.skipTests = value.skipTests;
     if (pattern) {
       this.pattern = this.transformPattern(pattern);
     }
@@ -300,6 +301,7 @@ class WPTTestSpec {
     this.failedTests = [];
     this.flakyTests = [];
     this.skipReasons = [];
+    this.skippedTests = [];
     for (const item of rules) {
       if (item.requires.length) {
         for (const req of item.requires) {
@@ -316,6 +318,9 @@ class WPTTestSpec {
       if (item.skip) {
         this.skipReasons.push(item.skip);
       }
+      if (Array.isArray(item.skipTests)) {
+        this.skippedTests.push(...item.skipTests);
+      }
     }
 
     this.failedTests = [...new Set(this.failedTests)];
@@ -334,6 +339,22 @@ class WPTTestSpec {
     return meta.variant?.map((variant) => new WPTTestSpec(mod, filename, rules, variant)) || [spec];
   }
 
+  /**
+   * Check if a subtest should be skipped by name.
+   * @param {string} name
+   * @returns {boolean}
+   */
+  isSkippedTest(name) {
+    for (const matcher of this.skippedTests) {
+      if (typeof matcher === 'string') {
+        if (name === matcher) return true;
+      } else if (matcher.test(name)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   getRelativePath() {
     return path.join(this.module, this.filename);
   }
@@ -669,6 +690,7 @@ class WPTRunner {
             },
             scriptsToRun,
             needsGc: !!meta.script?.find((script) => script === '/common/gc.js'),
+            skippedTests: spec.skippedTests,
           },
         });
         this.inProgress.add(spec);
@@ -679,6 +701,8 @@ class WPTRunner {
           switch (message.type) {
             case 'result':
               return this.resultCallback(spec, message.result, reportResult);
+            case 'skip':
+              return this.skipTest(spec, { name: message.name }, reportResult);
             case 'completion':
               return this.completionCallback(spec, message.status, reportResult);
             default:
@@ -736,6 +760,7 @@ class WPTRunner {
       const failures = [];
       let expectedFailures = 0;
       let skipped = 0;
+      let skippedTests = 0;
       for (const [key, item] of Object.entries(this.results)) {
         if (item.fail?.unexpected) {
           failures.push(key);
@@ -746,6 +771,9 @@ class WPTRunner {
         if (item.skip) {
           skipped++;
         }
+        if (item.skipTests) {
+          skippedTests += item.skipTests.length;
+        }
       }
 
       const unexpectedPasses = [];
@@ -786,7 +814,8 @@ class WPTRunner {
       console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
                   `${passed} passed, ${expectedFailures} expected failures,`,
                   `${failures.length} unexpected failures,`,
-                  `${unexpectedPasses.length} unexpected passes`);
+                  `${unexpectedPasses.length} unexpected passes` +
+                  (skippedTests ? `, ${skippedTests} subtests skipped` : ''));
       if (failures.length > 0) {
         const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
         throw new Error(
@@ -875,8 +904,16 @@ class WPTRunner {
     let result = this.results[spec.filename];
     result ||= this.results[spec.filename] = {};
     if (item.status === kSkip) {
-      // { filename: { skip: 'reason' } }
-      result[kSkip] = item.reason;
+      if (item.name) {
+        // Subtest-level skip: { filename: { skipTests: [ ... ] } }
+        result.skipTests ||= [];
+        if (!result.skipTests.includes(item.name)) {
+          result.skipTests.push(item.name);
+        }
+      } else {
+        // File-level skip: { filename: { skip: 'reason' } }
+        result[kSkip] = item.reason;
+      }
     } else {
       // { filename: { fail: { expected: [ ... ],
       //                      unexpected: [ ... ] } }}
@@ -895,6 +932,15 @@ class WPTRunner {
     reportResult?.addSubtest(test.name, 'PASS');
   }
 
+  skipTest(spec, test, reportResult) {
+    console.log(`[SKIP] ${test.name}`);
+    reportResult?.addSubtest(test.name, 'NOTRUN');
+    this.addTestResult(spec, {
+      name: test.name,
+      status: kSkip,
+    });
+  }
+
   fail(spec, test, status, reportResult) {
     const expected = spec.failedTests.includes(test.name);
     if (expected) {

test/common/wpt/worker.js

@@ -35,6 +35,32 @@ runInThisContext(workerData.harness.code, {
   filename: workerData.harness.filename,
 });
 
+// If there are skip patterns, wrap test functions to prevent execution of
+// matching tests. This must happen after testharness.js is loaded but before
+// the test scripts run.
+if (workerData.skippedTests?.length) {
+  function isSkipped(name) {
+    for (const matcher of workerData.skippedTests) {
+      if (typeof matcher === 'string') {
+        if (name === matcher) return true;
+      } else if (matcher.test(name)) {
+        return true;
+      }
+    }
+    return false;
+  }
+  for (const fn of ['test', 'async_test', 'promise_test']) {
+    const original = globalThis[fn];
+    globalThis[fn] = function(func, name, ...rest) {
+      if (typeof name === 'string' && isSkipped(name)) {
+        parentPort.postMessage({ type: 'skip', name });
+        return;
+      }
+      return original.call(this, func, name, ...rest);
+    };
+  }
+}
+
 // eslint-disable-next-line no-undef
 add_result_callback((result) => {
   parentPort.postMessage({

test/wpt/README.md

@@ -155,7 +155,7 @@ expected failures.
     // Optional: If the requirement is not met, this test will be skipped
     "requires": ["small-icu"],  // supports: "small-icu", "full-icu", "crypto"
 
-    // Optional: the test will be skipped with the reason printed
+    // Optional: the entire file will be skipped with the reason printed
     "skip": "explain why we cannot run a test that's supposed to pass",
 
     // Optional: failing tests
@@ -173,6 +173,42 @@ expected failures.
 }
 ```
 
+### Skipping individual subtests
+
+To skip specific subtests within a file (rather than skipping the entire file),
+use `skipTests` with an array of exact test names:
+
+```json
+{
+  "something.scope.js": {
+    "skipTests": [
+      "exact test name to skip"
+    ]
+  }
+}
+```
+
+When the status file is a CJS module, regular expressions can also be used:
+
+```js
+module.exports = {
+  'something.scope.js': {
+    'skipTests': [
+      'exact test name to skip',
+      /regexp pattern to match/,
+    ],
+  },
+};
+```
+
+Skipped subtests are reported as `[SKIP]` in the output, recorded as `NOTRUN`
+in the WPT report, and counted separately in the summary line.
+
+This is useful for skipping a particular subtest that crashes the runner,
+which would otherwise prevent the rest of the file from being run. When using
+CJS status files, this also enables conditionally skipping slow or
+resource-heavy subtests in CI on specific architectures.
+
 A test may have to be skipped because it depends on another irrelevant
 Web API, or certain harness has not been ported in our test runner yet.
 In that case it needs to be marked with `skip` instead of `fail`.