[FLINK-29198][test] Fail after maximum RetryOnException

Co-authored-by: Sergey Nuyanzin <[email protected]>
mbalassi · Nov 4, 2022 · 7339690 · 7339690
1 parent cf3158b
commit 7339690
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 8 deletions.
diff --git a/.../org/apache/flink/testutils/junit/extensions/retry/strategy/RetryOnExceptionStrategy.java b/.../org/apache/flink/testutils/junit/extensions/retry/strategy/RetryOnExceptionStrategy.java
@@ -22,7 +22,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/** Retry strategy that retry fixed times, and will not fail with some kind of exception. */
+/**
+ * A retry strategy that will ignore a specific type of exception and retry a test if it occurs, up
+ * to a fixed number of times.
+ */
 public class RetryOnExceptionStrategy extends AbstractRetryStrategy {
  private static final Logger LOG = LoggerFactory.getLogger(RetryOnExceptionStrategy.class);
 
@@ -37,6 +40,12 @@ public RetryOnExceptionStrategy(
  @Override
  public void handleException(String testName, int attemptIndex, Throwable throwable)
  throws Throwable {
+ // Failed when reach the total retry times
+ if (attemptIndex >= totalTimes) {
+ LOG.error("Test Failed at the last retry.", throwable);
+ throw throwable;
+ }
+
  if (repeatableException.isAssignableFrom(throwable.getClass())) {
  // continue retrying when get some repeatable exceptions
  String retryMsg =

diff --git a/...s-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnExceptionExtensionTest.java b/...s-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnExceptionExtensionTest.java
@@ -19,14 +19,21 @@
 package org.apache.flink.testutils.junit;
 
 import org.apache.flink.testutils.junit.extensions.retry.RetryExtension;
+import org.apache.flink.testutils.junit.extensions.retry.strategy.RetryOnExceptionStrategy;
 
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.opentest4j.TestAbortedException;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import java.util.stream.Stream;
 
-/** Tests for the RetryOnException annotation. */
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for the {@link RetryOnException} annotation on JUnit5 {@link RetryExtension}. */
 @ExtendWith(RetryExtension.class)
 class RetryOnExceptionExtensionTest {
 
@@ -42,10 +49,10 @@ class RetryOnExceptionExtensionTest {
 
  @AfterAll
  static void verify() {
- assertEquals(NUMBER_OF_RUNS + 1, runsForTestWithMatchingException);
- assertEquals(NUMBER_OF_RUNS + 1, runsForTestWithSubclassException);
- assertEquals(1, runsForSuccessfulTest);
- assertEquals(2, runsForPassAfterOneFailure);
+ assertThat(runsForTestWithMatchingException).isEqualTo(NUMBER_OF_RUNS + 1);
+ assertThat(runsForTestWithSubclassException).isEqualTo(NUMBER_OF_RUNS + 1);
+ assertThat(runsForSuccessfulTest).isOne();
+ assertThat(runsForPassAfterOneFailure).isEqualTo(2);
  }
 
  @TestTemplate
@@ -80,4 +87,42 @@ void testPassAfterOneFailure() {
  throw new IllegalArgumentException();
  }
  }
+
+ @ParameterizedTest(name = "Retrying with {0}")
+ @MethodSource("retryTestProvider")
+ void testRetryFailsWithExpectedExceptionAfterNumberOfRetries(
+ final Throwable expectedException) {
+ final int numberOfRetries = 1;
+ RetryOnExceptionStrategy retryOnExceptionStrategy =
+ new RetryOnExceptionStrategy(numberOfRetries, expectedException.getClass());
+ // All attempts that permit a retry should be a TestAbortedException. When retries are no
+ // longer permitted, the handled exception should be propagated.
+ for (int j = 0; j <= numberOfRetries; j++) {
+ final int attemptIndex = j;
+ assertThatThrownBy(
+ () ->
+ retryOnExceptionStrategy.handleException(
+ "Any test name", attemptIndex, expectedException))
+ .isInstanceOf(
+ j == numberOfRetries
+ ? expectedException.getClass()
+ : TestAbortedException.class);
+ }
+ }
+
+ static class RetryTestError extends Error {}
+
+ static class RetryTestException extends Exception {}
+
+ static class RetryTestRuntimeException extends RuntimeException {}
+
+ static class RetryTestThrowable extends Throwable {}
+
+ static Stream<Throwable> retryTestProvider() {
+ return Stream.of(
+ new RetryTestError(),
+ new RetryTestException(),
+ new RetryTestRuntimeException(),
+ new RetryTestThrowable());
+ }
 }
diff --git a/...ils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnFailureExtensionTest.java b/...ils-junit/src/test/java/org/apache/flink/testutils/junit/RetryOnFailureExtensionTest.java
@@ -26,7 +26,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-/** Tests for the RetryOnFailure annotation. */
+/** Tests for the {@link RetryOnFailure} annotation on JUnit5 {@link RetryExtension}. */
 @ExtendWith(RetryExtension.class)
 class RetryOnFailureExtensionTest {