fix(test): output parallel test results independently (denoland#15399)

.github/workflows/ci.yml

@@ -235,7 +235,7 @@ jobs:
  ~/.cargo/registry/index
  ~/.cargo/registry/cache
  ~/.cargo/git/db
- key: 17-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
+ key: 18-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
 
  # In main branch, always creates fresh cache
  - name: Cache build output (main)
@@ -251,7 +251,7 @@ jobs:
  !./target/*/*.zip
  !./target/*/*.tar.gz
  key: |
- 17-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
+ 18-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
 
  # Restore cache from the latest 'main' branch build.
  - name: Cache build output (PR)
@@ -267,7 +267,7 @@ jobs:
  !./target/*/*.tar.gz
  key: never_saved
  restore-keys: |
- 17-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
+ 18-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
 
  # Don't save cache after building PRs or branches other than 'main'.
  - name: Skip save cache (PR)

cli/lsp/testing/execution.rs

@@ -374,13 +374,12 @@ impl TestRun {
  .buffer_unordered(concurrent_jobs)
  .collect::<Vec<Result<Result<(), AnyError>, tokio::task::JoinError>>>();
 
- let mut reporter: Box<dyn test::TestReporter + Send> =
- Box::new(LspTestReporter::new(
- self,
- client.clone(),
- maybe_root_uri,
- self.tests.clone(),
- ));
+ let mut reporter = Box::new(LspTestReporter::new(
+ self,
+ client.clone(),
+ maybe_root_uri,
+ self.tests.clone(),
+ ));
 
  let handler = {
  tokio::task::spawn(async move {
@@ -653,9 +652,7 @@ impl LspTestReporter {
  },
  ));
  }
-}
 
-impl test::TestReporter for LspTestReporter {
  fn report_plan(&mut self, _plan: &test::TestPlan) {}
 
  fn report_register(&mut self, desc: &test::TestDescription) {

cli/tests/integration/test_tests.rs

@@ -304,12 +304,6 @@ itest!(steps_passing_steps {
  output: "test/steps/passing_steps.out",
 });
 
-itest!(steps_passing_steps_concurrent {
- args: "test --jobs=2 test/steps/passing_steps.ts",
- exit_code: 0,
- output: "test/steps/passing_steps.out",
-});
-
 itest!(steps_failing_steps {
  args: "test test/steps/failing_steps.ts",
  exit_code: 1,
@@ -447,3 +441,9 @@ itest!(non_error_thrown {
  output: "test/non_error_thrown.out",
  exit_code: 1,
 });
+
+itest!(parallel_output {
+ args: "test --parallel --reload test/parallel_output.ts",
+ output: "test/parallel_output.out",
+ exit_code: 1,
+});

cli/tests/testdata/test/parallel_output.out

@@ -0,0 +1,57 @@
+Check [WILDCARD]/test/parallel_output.ts
+./test/parallel_output.ts => step output ... step 1 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step output ... step 2 ... ok ([WILDCARD]ms)
+------- output -------
+Hello, world! (from step 3)
+----- output end -----
+./test/parallel_output.ts => step output ... step 3 ... ok ([WILDCARD]ms)
+------- output -------
+Hello, world! (from step 4)
+----- output end -----
+./test/parallel_output.ts => step output ... step 4 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step output ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step failures ... step 1 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step failures ... step 2 ... FAILED ([WILDCARD]ms)
+ error: Error: Fail.
+ throw new Error("Fail.");
+ ^
+ at file:https:///[WILDCARD]/test/parallel_output.ts:15:11
+ at [WILDCARD]
+ at file:https:///[WILDCARD]/test/parallel_output.ts:14:11
+./test/parallel_output.ts => step failures ... step 3 ... FAILED ([WILDCARD]ms)
+ error: Error: Fail.
+ await t.step("step 3", () => Promise.reject(new Error("Fail.")));
+ ^
+ at file:https:///[WILDCARD]/test/parallel_output.ts:17:47
+ at [WILDCARD]
+ at file:https:///[WILDCARD]/test/parallel_output.ts:17:11
+./test/parallel_output.ts => step failures ... FAILED ([WILDCARD]ms)
+./test/parallel_output.ts => step nested failure ... step 1 ... inner 1 ... ok ([WILDCARD]ms)
+./test/parallel_output.ts => step nested failure ... step 1 ... inner 2 ... FAILED ([WILDCARD]ms)
+ error: Error: Failed.
+ throw new Error("Failed.");
+ ^
+ at file:https:///[WILDCARD]/test/parallel_output.ts:24:13
+ at [WILDCARD]
+ at file:https:///[WILDCARD]/test/parallel_output.ts:23:13
+./test/parallel_output.ts => step nested failure ... step 1 ... FAILED ([WILDCARD]ms)
+./test/parallel_output.ts => step nested failure ... FAILED ([WILDCARD]ms)
+
+ ERRORS 
+
+step failures => ./test/parallel_output.ts:12:6
+error: Error: 2 test steps failed.
+ at [WILDCARD]
+
+step nested failure => ./test/parallel_output.ts:20:6
+error: Error: 1 test step failed.
+ at [WILDCARD]
+
+ FAILURES 
+
+step failures => ./test/parallel_output.ts:12:6
+step nested failure => ./test/parallel_output.ts:20:6
+
+FAILED | 1 passed (6 steps) | 2 failed (4 steps) ([WILDCARD]ms)
+
+error: Test failed

cli/tests/testdata/test/parallel_output.ts

@@ -0,0 +1,27 @@
+Deno.test("step output", async (t) => {
+ await t.step("step 1", () => {});
+ await t.step("step 2", () => {});
+ await t.step("step 3", () => {
+ console.log("Hello, world! (from step 3)");
+ });
+ await t.step("step 4", () => {
+ console.log("Hello, world! (from step 4)");
+ });
+});
+
+Deno.test("step failures", async (t) => {
+ await t.step("step 1", () => {});
+ await t.step("step 2", () => {
+ throw new Error("Fail.");
+ });
+ await t.step("step 3", () => Promise.reject(new Error("Fail.")));
+});
+
+Deno.test("step nested failure", async (t) => {
+ await t.step("step 1", async (t) => {
+ await t.step("inner 1", () => {});
+ await t.step("inner 2", () => {
+ throw new Error("Failed.");
+ });
+ });
+});

cli/tests/testdata/test/short-pass-jobs-flag-warning.out

@@ -1,7 +1,6 @@
 Warning: --jobs flag is deprecated. Use the --parallel flag with possibly the 'DENO_JOBS' environment variable.
 Check [WILDCARD]/test/short-pass.ts
-running 1 test from ./test/short-pass.ts
-test ... ok ([WILDCARD])
+./test/short-pass.ts => test ... ok ([WILDCARD])
 
 ok | 1 passed | 0 failed ([WILDCARD])
 

cli/tests/testdata/test/short-pass.out

@@ -1,6 +1,5 @@
 Check [WILDCARD]/test/short-pass.ts
-running 1 test from ./test/short-pass.ts
-test ... ok ([WILDCARD])
+./test/short-pass.ts => test ... ok ([WILDCARD])
 
 ok | 1 passed | 0 failed ([WILDCARD])
 

cli/tests/testdata/test/steps/output_within.out

@@ -18,12 +18,10 @@ description ...
 4
 ----- output end -----
  inner 2 ... ok ([WILDCARD]ms)
-
 ------- output -------
 5
 ----- output end -----
  step 1 ... ok ([WILDCARD]ms)
-
 ------- output -------
 6
 ----- output end -----