[FLINK-10801][e2e] Retry verify_result_hash in elastichsearch-common (a…

…pache#7060)

Instead of looping the verification until the expected number of results
loop until we get the correct output. This tries to solve the problem
of some records (aggregated? updated?) arriving later.

flink-end-to-end-tests/test-scripts/common.sh

@@ -451,6 +451,16 @@ function cancel_job {
 }
 
 function check_result_hash {
+ local error_code=0
+ check_result_hash_no_exit "$@" || error_code=$?
+
+ if [ "$error_code" != "0" ]
+ then
+ exit $error_code
+ fi
+}
+
+function check_result_hash_no_exit {
  local name=$1
  local outfile_prefix=$2
  local expected=$3
@@ -462,18 +472,19 @@ function check_result_hash {
  actual=$(LC_ALL=C sort $outfile_prefix* | md5sum | awk '{print $1}')
  else
  echo "Neither 'md5' nor 'md5sum' binary available."
- exit 2
+ return 2
  fi
  if [[ "$actual" != "$expected" ]]
  then
  echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
  echo "head hexdump of actual:"
  head $outfile_prefix* | hexdump -c
- exit 1
+ return 1
  else
  echo "pass $name"
  # Output files are left behind in /tmp
  fi
+ return 0
 }
 
 # This function starts the given number of task managers and monitors their processes.

flink-end-to-end-tests/test-scripts/elasticsearch-common.sh

@@ -83,23 +83,34 @@ function verify_result_hash {
  local name=$1
  local index=$2
  local numRecords=$3
- local hash=$4
+ local expectedHash=$4
 
- while : ; do
+ for i in {1..30}
+ do
+ local error_code=0
+
+ echo "Result verification attempt $i..."
  curl "localhost:9200/${index}/_search?q=*&pretty" > $TEST_DATA_DIR/es_output || true
 
- if [ -n "$(grep "\"total\" : $numRecords" $TEST_DATA_DIR/es_output)" ]; then
- break
- else
- echo "Waiting for Elasticsearch records ..."
+ # remove meta information
+ sed '2,9d' $TEST_DATA_DIR/es_output > $TEST_DATA_DIR/es_content
+
+ check_result_hash_no_exit "$name" $TEST_DATA_DIR/es_content "$expectedHash" || error_code=$?
+
+ if [ "$error_code" != "0" ]
+ then
+ echo "Result verification attempt $i has failed"
  sleep 1
+ else
+ break
  fi
  done
 
- # remove meta information
- sed '2,9d' $TEST_DATA_DIR/es_output > $TEST_DATA_DIR/es_content
-
- check_result_hash "$name" $TEST_DATA_DIR/es_content "$hash"
+ if [ "$error_code" != "0" ]
+ then
+ echo "All result verification attempts have failed"
+ exit $error_code
+ fi
 }
 
 function shutdown_elasticsearch_cluster {