[FLINK-10711] [e2e] Allow basic error handling with bash

This commit modifies the test infrastructure to allow bash's basic
error handling mechanism with set -e. Many tests are not
ready for a globally defined strict error handling. For now, at
least newly developed tests should consider this flag. If a test
causes an error, a test fails with "[FAIL] Test script contains errors".

See the end-to-end test README for more information about how to develop
tests in the future.

This closes apache#7023.

flink-end-to-end-tests/README.md

@@ -47,6 +47,8 @@ In order to add a new test case you need add it to either `test-scripts/run-nigh
 
 _Note: If you want to parameterize your tests please do so by adding multiple test cases with parameters as arguments to the nightly / pre-commit test suites. This allows the test runner to do a cleanup in between each individual test and also to fail those tests individually._
 
+_Note: While developing a new test case make sure to enable bash's error handling in `test-scripts/common.sh` by uncommenting `set -Eexuo pipefail` and commenting the current default `set` call. Once your test is implemented properly, add `set -Eeuo pipefail` on the very top of your test script (before any `common` script)._
+
 ### Passing your test
 A test is considered to have passed if it:
 - has exit code 0

flink-end-to-end-tests/test-scripts/common.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 ################################################################################
 
+# Enable this line when developing a new end-to-end test
+#set -Eexuo pipefail
 set -o pipefail
 
 if [[ -z $FLINK_DIR ]]; then
@@ -304,6 +306,7 @@ function start_and_wait_for_tm {
 }
 
 function check_logs_for_errors {
+ echo "Checking for errors..."
  error_count=$(grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \
  | grep -v "RetriableCommitFailedException" \
  | grep -v "NoAvailableBrokersException" \
@@ -321,15 +324,18 @@ function check_logs_for_errors {
  | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration" \
  | grep -v "org.apache.flink.fs.shaded.hadoop3.org.apache.commons.beanutils.FluentPropertyBeanIntrospector - Error when creating PropertyDescriptor for public final void org.apache.flink.fs.shaded.hadoop3.org.apache.commons.configuration2.AbstractConfiguration.setProperty(java.lang.String,java.lang.Object)! Ignoring this property." \
  | grep -v "Error while loading kafka-version.properties :null" \
- | grep -ic "error")
+ | grep -ic "error" || true)
  if [[ ${error_count} -gt 0 ]]; then
  echo "Found error in log files:"
  cat $FLINK_DIR/log/*
  EXIT_CODE=1
+ else
+ echo "No errors in log files."
  fi
 }
 
 function check_logs_for_exceptions {
+ echo "Checking for exceptions..."
  exception_count=$(grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \
  | grep -v "RetriableCommitFailedException" \
  | grep -v "NoAvailableBrokersException" \
@@ -349,19 +355,24 @@ function check_logs_for_exceptions {
  | grep -v "java.io.InvalidClassException: org.apache.flink.formats.avro.typeutils.AvroSerializer" \
  | grep -v "Caused by: java.lang.Exception: JobManager is shutting down" \
  | grep -v "java.lang.Exception: Artificial failure" \
- | grep -ic "exception")
+ | grep -ic "exception" || true)
  if [[ ${exception_count} -gt 0 ]]; then
  echo "Found exception in log files:"
  cat $FLINK_DIR/log/*
  EXIT_CODE=1
+ else
+ echo "No exceptions in log files."
  fi
 }
 
 function check_logs_for_non_empty_out_files {
+ echo "Checking for non-empty .out files..."
  if grep -ri "." $FLINK_DIR/log/*.out > /dev/null; then
  echo "Found non-empty .out files:"
  cat $FLINK_DIR/log/*.out
  EXIT_CODE=1
+ else
+ echo "No non-empty .out files."
  fi
 }
 
@@ -375,7 +386,9 @@ function stop_cluster {
  "$FLINK_DIR"/bin/stop-cluster.sh
 
  # stop zookeeper only if there are processes running
- if ! [ "`jps | grep 'FlinkZooKeeperQuorumPeer' | wc -l`" = "0" ]; then
+ zookeeper_process_count=$(jps | grep -c 'FlinkZooKeeperQuorumPeer' || true)
+ if [[ ${zookeeper_process_count} -gt 0 ]]; then
+ echo "Stopping zookeeper..."
  "$FLINK_DIR"/bin/zookeeper.sh stop
  fi
 }
@@ -490,9 +503,9 @@ function tm_kill_all {
 
 # Kills all processes that match the given name.
 function kill_all {
- local pid=`jps | grep -E "${1}" | cut -d " " -f 1`
- kill ${pid} 2> /dev/null
- wait ${pid} 2> /dev/null
+ local pid=`jps | grep -E "${1}" | cut -d " " -f 1 || true`
+ kill ${pid} 2> /dev/null || true
+ wait ${pid} 2> /dev/null || true
 }
 
 function kill_random_taskmanager {

flink-end-to-end-tests/test-scripts/elasticsearch-common.sh

@@ -17,8 +17,6 @@
 # limitations under the License.
 ################################################################################
 
-set -o pipefail
-
 if [[ -z $TEST_DATA_DIR ]]; then
  echo "Must run common.sh before elasticsearch-common.sh."
  exit 1

flink-end-to-end-tests/test-scripts/kafka-common.sh

@@ -17,10 +17,6 @@
 # limitations under the License.
 ################################################################################
 
-set -e
-set -u
-set -o pipefail
-
 if [[ -z $TEST_DATA_DIR ]]; then
  echo "Must run common.sh before kafka-common.sh."
  exit 1

flink-end-to-end-tests/test-scripts/test-runner-common.sh

@@ -29,7 +29,7 @@ source "${END_TO_END_DIR}"/test-scripts/common.sh
 function run_test {
  local description="$1"
  local command="$2"
- local skip_check_exceptions="$3"
+ local skip_check_exceptions=${3:-}
 
  printf "\n==============================================================================\n"
  printf "Running '${description}'\n"
@@ -41,14 +41,32 @@ function run_test {
 
  backup_config
  start_timer
+
+ function test_error() {
+ echo "[FAIL] Test script contains errors."
+ post_test_validation 1 "$description" "$skip_check_exceptions"
+ }
+ trap 'test_error' ERR
+
  ${command}
  exit_code="$?"
- time_elapsed=$(end_timer)
+ post_test_validation ${exit_code} "$description" "$skip_check_exceptions"
+}
+
+# Validates the test result and exit code after its execution.
+function post_test_validation {
+ local exit_code="$1"
+ local description="$2"
+ local skip_check_exceptions="$3"
+
+ local time_elapsed=$(end_timer)
 
  if [[ "${skip_check_exceptions}" != "skip_check_exceptions" ]]; then
  check_logs_for_errors
  check_logs_for_exceptions
  check_logs_for_non_empty_out_files
+ else
+ echo "Checking of logs skipped."
  fi
 
  # Investigate exit_code for failures of test executable as well as EXIT_CODE for failures of the test.

flink-end-to-end-tests/test-scripts/test_batch_allround.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 ################################################################################
 
+set -Eeuo pipefail
+
 source "$(dirname "$0")"/common.sh
 
 TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-dataset-allround-test/target/DataSetAllroundTestProgram.jar

flink-end-to-end-tests/test-scripts/test_cli.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 ################################################################################
 
+set -Eeuo pipefail
+
 source "$(dirname "$0")"/common.sh
 
 TEST_PROGRAM_JAR=$END_TO_END_DIR/flink-cli-test/target/PeriodicStreamingJob.jar

flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh

@@ -17,9 +17,7 @@
 # limitations under the License.
 ################################################################################
 
-set -e
-set -u
-set -o pipefail
+set -Eeuo pipefail
 
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka-common.sh 2.0.0 5.0.0 5.0

flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 ################################################################################
 
+set -Eeuo pipefail
+
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka-common.sh 0.10.2.0 3.2.0 3.2
 

flink-end-to-end-tests/test-scripts/test_streaming_kafka_common.sh

@@ -17,10 +17,6 @@
 # limitations under the License.
 ################################################################################
 
-set -e
-set -u
-set -o pipefail
-
 KAFKA_EXAMPLE_JAR="$1"
 
 setup_kafka_dist