From a83eb8471e81ca3e5ffe83471a6080b31b12d1d3 Mon Sep 17 00:00:00 2001 From: Andrew Parmet Date: Sat, 6 Jan 2024 14:19:13 -0500 Subject: [PATCH 1/3] improve conformance test failure reporting --- .../protokt/v1/conformance/ConformanceTest.kt | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt b/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt index c4f3134e0..04ef222a0 100644 --- a/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt +++ b/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt @@ -15,14 +15,18 @@ package protokt.v1.conformance +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.fail import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource +import org.junit.jupiter.params.provider.EnumSource import protokt.v1.testing.ProcessOutput import protokt.v1.testing.projectRoot import protokt.v1.testing.runCommand import java.io.File import java.nio.file.Path +import kotlin.io.path.deleteIfExists +import kotlin.io.path.exists +import kotlin.io.path.readText class ConformanceTest { enum class ConformanceRunner( @@ -37,9 +41,9 @@ class ConformanceTest { override fun driver() = jsConformanceDriver(project) - override fun handle(t: Throwable) { + override fun doHandle(t: Throwable) { val stderr = jsStderrLog(project).toFile() - if (stderr.exists()) { + if (stderr.exists() && stderr.readText().isNotEmpty()) { fail("test failed; stderr:\n" + stderr.readText()) } else { throw t @@ -50,19 +54,25 @@ class ConformanceTest { abstract fun driver(): Path - open fun handle(t: Throwable) { + fun handle(t: Throwable) { + if (failingTests.exists()) { + println("Failing tests:\n" + failingTests.readText()) + } + doHandle(t) + } + + open fun doHandle(t: Throwable) { throw t } } - companion object { - @JvmStatic - fun runners() = - ConformanceRunner.entries + @BeforeEach + fun deleteFailingTests() { + failingTests.deleteIfExists() } @ParameterizedTest - @MethodSource("runners") + @EnumSource fun `run conformance tests`(runner: ConformanceRunner) { try { command(runner) @@ -100,6 +110,9 @@ private fun jsConformanceDriver(project: String) = private fun jsStderrLog(project: String) = Path.of(File(projectRoot.parentFile, project).absolutePath, "build", "conformance-run") +private val failingTests = + Path.of(projectRoot.absolutePath, "failing_tests.txt") + private fun failureList(project: String) = "--failure_list ../$project/failure_list_kt.txt" From 666b41ea083fb1225e21ebe4401c3a501a018ef0 Mon Sep 17 00:00:00 2001 From: Andrew Parmet Date: Sat, 6 Jan 2024 14:24:13 -0500 Subject: [PATCH 2/3] spurious pass --- .../protokt/v1/conformance/ConformanceTest.kt | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt b/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt index 04ef222a0..b547438d2 100644 --- a/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt +++ b/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt @@ -41,12 +41,10 @@ class ConformanceTest { override fun driver() = jsConformanceDriver(project) - override fun doHandle(t: Throwable) { + override fun onFailure() { val stderr = jsStderrLog(project).toFile() if (stderr.exists() && stderr.readText().isNotEmpty()) { - fail("test failed; stderr:\n" + stderr.readText()) - } else { - throw t + println("stderr:\n" + stderr.readText()) } } }, // https://github.com/pinterest/ktlint/issues/1933 @@ -58,12 +56,11 @@ class ConformanceTest { if (failingTests.exists()) { println("Failing tests:\n" + failingTests.readText()) } - doHandle(t) - } - - open fun doHandle(t: Throwable) { + onFailure() throw t } + + open fun onFailure() = Unit } @BeforeEach From 945bc4fe79b34f44aef8475663a927108ecd2360 Mon Sep 17 00:00:00 2001 From: Andrew Parmet Date: Sat, 6 Jan 2024 14:26:34 -0500 Subject: [PATCH 3/3] better --- .../protokt/v1/conformance/ConformanceTest.kt | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt b/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt index b547438d2..16b31e8c6 100644 --- a/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt +++ b/testing/conformance/runner/src/test/kotlin/protokt/v1/conformance/ConformanceTest.kt @@ -52,14 +52,6 @@ class ConformanceTest { abstract fun driver(): Path - fun handle(t: Throwable) { - if (failingTests.exists()) { - println("Failing tests:\n" + failingTests.readText()) - } - onFailure() - throw t - } - open fun onFailure() = Unit } @@ -76,7 +68,11 @@ class ConformanceTest { .runCommand(projectRoot.toPath(), libPathOverride) .orFail("Conformance tests failed", ProcessOutput.Src.ERR) } catch (t: Throwable) { - runner.handle(t) + if (failingTests.exists()) { + println("Failing tests:\n" + failingTests.readText()) + } + runner.onFailure() + throw t } println("Conformance tests passed")