[FLINK-10756][runtime][tests] Wait for TM processes to shutdown

flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java

@@ -41,9 +41,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.apache.flink.runtime.testutils.CommonTestUtils.getJavaCommandPath;
@@ -213,6 +216,22 @@ public void run() {
  if (taskManagerProcess3 != null) {
  taskManagerProcess3.destroy();
  }
+
+ waitForShutdown("TaskManager 1", taskManagerProcess1);
+ waitForShutdown("TaskManager 2", taskManagerProcess2);
+ waitForShutdown("TaskManager 3", taskManagerProcess3);
+ }
+ }
+
+ private void waitForShutdown(final String processName, @Nullable final TestProcess process) throws InterruptedException {
+ if (process == null) {
+ return;
+ }
+
+ if (!process.getProcess().waitFor(30, TimeUnit.SECONDS)) {
+ log.error("{} did not shutdown in time.", processName);
+ printProcessLog(processName, process.getOutput().toString());
+ process.getProcess().destroyForcibly();
  }
  }