test(wpt): implement process timeout, fix expectations update, and mo…

…re... (denoland#17892)

- relands denoland#17872
- updates the timeouts to be re-configurable just for CI
- fixes `./tools/wpt.ts update`
- adds option not "ignore" during, applied to wpt epoch runs only

.github/workflows/wpt_epoch.yml

@@ -72,7 +72,7 @@ jobs:
  deno run --unstable --allow-write --allow-read --allow-net \
  --allow-env --allow-run --lock=tools/deno.lock.json \
  ./tools/wpt.ts run \
- --binary=$(which deno) --quiet --release --json=wpt.json --wptreport=wptreport.json || true
+ --binary=$(which deno) --quiet --release --no-ignore --json=wpt.json --wptreport=wptreport.json || true
 
  - name: Upload wpt results to wpt.fyi
  env:

tools/wpt.ts

@@ -25,6 +25,7 @@ import {
  ManifestFolder,
  ManifestTestOptions,
  ManifestTestVariation,
+ noIgnore,
  quiet,
  rest,
  runPy,
@@ -173,6 +174,9 @@ async function run() {
  test.options,
  inParallel ? () => {} : createReportTestCase(test.expectation),
  inspectBrk,
+ Deno.env.get("CI")
+ ? { long: 4 * 60_000, default: 4 * 60_000 }
+ : { long: 60_000, default: 10_000 },
  );
  results.push({ test, result });
  if (inParallel) {
@@ -332,6 +336,7 @@ async function update() {
  test.options,
  json ? () => {} : createReportTestCase(test.expectation),
  inspectBrk,
+ { long: 60_000, default: 10_000 },
  );
  results.push({ test, result });
  reportVariation(result, test.expectation);
@@ -367,7 +372,7 @@ async function update() {
 
  const currentExpectation = getExpectation();
 
- for (const result of Object.values(resultTests)) {
+ for (const [path, result] of Object.entries(resultTests)) {
  const { passed, failed, testSucceeded } = result;
  let finalExpectation: boolean | string[];
  if (failed.length == 0 && testSucceeded) {
@@ -699,7 +704,7 @@ function discoverTestsToRun(
  typeof expectation.ignore === "boolean",
  "test entry's `ignore` key must be a boolean",
  );
- if (expectation.ignore === true) continue;
+ if (expectation.ignore === true && !noIgnore) continue;
  }
  }
 

tools/wpt/expectation.json

@@ -1087,8 +1087,8 @@
  "EventTarget-constructible.any.html": true,
  "EventTarget-constructible.any.worker.html": true,
  "Event-constructors.any.html": [
- "Untitled 3",
- "Untitled 4"
+ "Event constructors 3",
+ "Event constructors 4"
  ],
  "Event-constructors.any.worker.html": [
  "Event constructors 3",

tools/wpt/runner.ts

@@ -75,13 +75,23 @@ export async function runSingleTest(
  _options: ManifestTestOptions,
  reporter: (result: TestCaseResult) => void,
  inspectBrk: boolean,
+ timeouts: { long: number; default: number },
 ): Promise<TestResult> {
+ const timeout = _options.timeout === "long"
+ ? timeouts.long
+ : timeouts.default;
+ const filename = url.pathname.substring(
+ url.pathname.lastIndexOf("/") + 1,
+ url.pathname.indexOf("."),
+ );
+ const { title } = Object.fromEntries(_options.script_metadata || []);
  const bundle = await generateBundle(url);
  const tempFile = await Deno.makeTempFile({
  prefix: "wpt-bundle-",
  suffix: ".js",
  });
 
+ let interval;
  try {
  await Deno.writeTextFile(tempFile, bundle);
 
@@ -107,6 +117,7 @@ export async function runSingleTest(
  "[]",
  );
 
+ const start = performance.now();
  const proc = new Deno.Command(denoBinary(), {
  args,
  env: {
@@ -124,10 +135,19 @@ export async function runSingleTest(
  const lines = proc.stderr.pipeThrough(new TextDecoderStream()).pipeThrough(
  new TextLineStream(),
  );
+ interval = setInterval(() => {
+ const passedTime = performance.now() - start;
+ if (passedTime > timeout) {
+ proc.kill("SIGINT");
+ }
+ }, 1000);
  for await (const line of lines) {
  if (line.startsWith("{")) {
  const data = JSON.parse(line);
  const result = { ...data, passed: data.status == 0 };
+ if (/^Untitled( \d+)?$/.test(result.name)) {
+ result.name = `${title || filename}${result.name.slice(8)}`;
+ }
  cases.push(result);
  reporter(result);
  } else if (line.startsWith("#$#$#{")) {
@@ -149,6 +169,7 @@ export async function runSingleTest(
  stderr,
  };
  } finally {
+ clearInterval(interval);
  await Deno.remove(tempFile);
  }
 }

tools/wpt/utils.ts

@@ -13,10 +13,11 @@ export const {
  ["--"]: rest,
  ["auto-config"]: autoConfig,
  ["inspect-brk"]: inspectBrk,
+ ["no-ignore"]: noIgnore,
  binary,
 } = parse(Deno.args, {
  "--": true,
- boolean: ["quiet", "release", "no-interactive", "inspect-brk"],
+ boolean: ["quiet", "release", "no-interactive", "inspect-brk", "no-ignore"],
  string: ["json", "wptreport", "binary"],
 });