[FLINK-10110][tests] Harden Kafka component shut down for E2E tests

Instead of only calling kafka-server-stop.sh and zookeeper-server-stop.sh which can fail
due to KAFKA-4931 we also use jps to kill the Kafka and ZooKeeper processes.

This closes apache#6530.

flink-end-to-end-tests/test-scripts/kafka-common.sh

@@ -75,6 +75,18 @@ function start_kafka_cluster {
 function stop_kafka_cluster {
  $KAFKA_DIR/bin/kafka-server-stop.sh
  $KAFKA_DIR/bin/zookeeper-server-stop.sh
+
+ PIDS=$(jps -vl | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}')
+
+ if [ ! -z "$PIDS" ]; then
+ kill -s TERM $PIDS
+ fi
+
+ PIDS=$(jps -vl | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')
+
+ if [ ! -z "$PIDS" ]; then
+ kill -s TERM $PIDS
+ fi
 }
 
 function create_kafka_topic {

flink-end-to-end-tests/test-scripts/test_confluent_schema_registry.sh

@@ -38,8 +38,8 @@ function test_cleanup {
  # don't call ourselves again for normal exit
  trap "" EXIT
 
- stop_kafka_cluster
  stop_confluent_schema_registry
+ stop_kafka_cluster
 }
 
 trap test_cleanup INT