[FLINK-10678][e2e] Introduced switch to disable log checking in e2e t…

…ests

flink-end-to-end-tests/run-nightly-tests.sh

@@ -41,19 +41,25 @@ echo "Flink distribution directory: $FLINK_DIR"
 
 # Template for adding a test:
 
-# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>" ["skip_check_exceptions"]
+
+# IMPORTANT:
+# With the "skip_check_exceptions" flag one can disable default exceptions and errors checking in log files. This should be done
+# carefully though. A valid reasons for doing so could be e.g killing TMs randomly as we cannot predict what exception could be thrown. Whenever
+# those checks are disabled, one should take care that a proper checks are performed in the tests itself that ensure that the test finished
+# in an expected state.
 
 run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25"
 
 run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb"
-run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh"
+run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh" "skip_check_exceptions"
 
-run_test "Running HA dataset end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_dataset.sh"
+run_test "Running HA dataset end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_dataset.sh" "skip_check_exceptions"
 
-run_test "Running HA (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file true false"
-run_test "Running HA (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file false false"
-run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false"
-run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true"
+run_test "Running HA (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file true false" "skip_check_exceptions"
+run_test "Running HA (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh file false false" "skip_check_exceptions"
+run_test "Running HA (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true false" "skip_check_exceptions"
+run_test "Running HA (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha_datastream.sh rocks true true" "skip_check_exceptions"
 
 run_test "Resuming Savepoint (file, async, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file true"
 run_test "Resuming Savepoint (file, sync, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file false"
@@ -87,17 +93,17 @@ run_test "Resuming Externalized Checkpoint after terminal failure (rocks, non-in
 run_test "Resuming Externalized Checkpoint after terminal failure (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true true true"
 
 run_test "DataSet allround end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_allround.sh"
-run_test "Streaming SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh"
-run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh"
-run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh"
+run_test "Streaming SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh" "skip_check_exceptions"
+run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh" "skip_check_exceptions"
+run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh" "skip_check_exceptions"
 run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
 
-run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false false"
-run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false true"
-run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false false"
-run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true false"
-run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false true"
-run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true true"
+run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false false" "skip_check_exceptions"
+run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 file false true" "skip_check_exceptions"
+run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false false" "skip_check_exceptions"
+run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true false" "skip_check_exceptions"
+run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false true" "skip_check_exceptions"
+run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true true" "skip_check_exceptions"
 
 run_test "Elasticsearch (v1.7.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 1 https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz"
 run_test "Elasticsearch (v2.3.5) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2 https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz"

flink-end-to-end-tests/run-pre-commit-tests.sh

@@ -40,7 +40,13 @@ echo "flink-end-to-end-test directory: $END_TO_END_DIR"
 echo "Flink distribution directory: $FLINK_DIR"
 
 # Template for adding a test:
-# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>" ["skip_check_exceptions"]
+
+# IMPORTANT:
+# With the "skip_check_exceptions" flag one can disable default exceptions and errors checking in log files. This should be done
+# carefully though. A valid reasons for doing so could be e.g killing TMs randomly as we cannot predict what exception could be thrown. Whenever
+# those checks are disabled, one should take care that a proper checks are performed in the tests itself that ensure that the test finished
+# in an expected state.
 
 run_test "State Migration end-to-end test from 1.6" "$END_TO_END_DIR/test-scripts/test_state_migration.sh"
 run_test "State Evolution end-to-end test" "$END_TO_END_DIR/test-scripts/test_state_evolution.sh"

flink-end-to-end-tests/run-single-test.sh

@@ -18,10 +18,16 @@
 ################################################################################
 
 # This can be used to run a single test in the context of a test runnner
-# Usage: ./run-single-test.sh test-scripts/my-test-case.sh
+# Usage: ./run-single-test.sh [skip] test-scripts/my-test-case.sh
+
+# IMPORTANT:
+# With the "skip" flag one can disable default exceptions and errors checking in log files. This should be done carefully though.
+# A valid reasons for doing so could be e.g killing TMs randomly as we cannot predict what exception could be thrown. Whenever
+# those checks are disabled, one should take care that a proper checks are performed in the tests itself that ensure that the test finished
+# in an expected state.
 
 if [ $# -eq 0 ]; then
- echo "Usage: ./run-single-test.sh <path-to-test-script> [<arg1> <arg2> ...]"
+ echo "Usage: ./run-single-test.sh [skip] <path-to-test-script> [<arg1> <arg2> ...]"
  exit 1
 fi
 
@@ -47,6 +53,10 @@ FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P )`" # absolutized and normalized
 echo "flink-end-to-end-test directory: $END_TO_END_DIR"
 echo "Flink distribution directory: $FLINK_DIR"
 
-run_test "$*" "$*"
+if [[ "$1" = "skip" ]]; then
+ run_test "${*:2}" "${*:2}" "skip_check_exceptions"
+else
+ run_test "$*" "$*"
+fi
 
 exit 0

flink-end-to-end-tests/test-scripts/test-runner-common.sh

@@ -24,10 +24,12 @@ source "${END_TO_END_DIR}"/test-scripts/common.sh
 # Arguments:
 # $1: description of the test
 # $2: command to execute
+# $3: check logs for erors & exceptions
 #######################################
 function run_test {
- description="$1"
- command="$2"
+ local description="$1"
+ local command="$2"
+ local skip_check_exceptions="$3"
 
  printf "\n==============================================================================\n"
  printf "Running '${description}'\n"
@@ -43,9 +45,11 @@ function run_test {
  exit_code="$?"
  time_elapsed=$(end_timer)
 
- check_logs_for_errors
- check_logs_for_exceptions
- check_logs_for_non_empty_out_files
+ if [[ "${skip_check_exceptions}" != "skip_check_exceptions" ]]; then
+ check_logs_for_errors
+ check_logs_for_exceptions
+ check_logs_for_non_empty_out_files
+ fi
 
  # Investigate exit_code for failures of test executable as well as EXIT_CODE for failures of the test.
  # Do not clean up if either fails.

flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh

@@ -51,10 +51,6 @@ function run_test() {
  local PARALLELISM=1 # parallelism of queryable state app
  local PORT="9069" # port of queryable state server
 
- # to ensure there are no files accidentally left behind by previous tests
- clean_log_files
- clean_stdout_files
-
  # speeds up TM loss detection
  set_conf "heartbeat.interval" "2000"
  set_conf "heartbeat.timeout" "10000"
@@ -169,12 +165,6 @@ function get_completed_number_of_checkpoints {
 
 function test_cleanup {
  unlink_queryable_state_lib
-
- # this is needed b.c. otherwise we might have exceptions from when
- # we kill the task manager left behind in the logs, which would cause
- # our test to fail in the cleanup function
- clean_log_files
- clean_stdout_files
 }
 
 trap test_cleanup EXIT

flink-end-to-end-tests/test-scripts/test_streaming_sql.sh

@@ -43,8 +43,6 @@ function sql_cleanup() {
  # remove flink-table from lib folder
  rm $FLINK_DIR/lib/flink-table*jar
 
- # remove logs cause they contain exceptions that shouldn't fail the test
- rm $FLINK_DIR/log/*
 }
 trap sql_cleanup INT
 trap sql_cleanup EXIT