[FLINK-10481][e2e] Added retry logic for building docker image

deepfield · Nov 19, 2018 · 868e411 · 868e411
1 parent ea9507d
commit 868e411
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 2 deletions.
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
@@ -663,3 +663,22 @@ function find_latest_completed_checkpoint {
  local checkpoint_meta_file=$(ls -d ${checkpoint_root_directory}/chk-[1-9]*/_metadata | sort -Vr | head -n1)
  echo "$(dirname "${checkpoint_meta_file}")"
 }
+
+function retry_times() {
+ local retriesNumber=$1
+ local backoff=$2
+ local command=${@:3}
+
+ for (( i = 0; i < ${retriesNumber}; i++ ))
+ do
+ if ${command}; then
+ return 0
+ fi
+
+ echo "Command: ${command} failed. Retrying..."
+ sleep ${backoff}
+ done
+
+ echo "Command: ${command} failed ${retriesNumber} times."
+ return 1
+}
diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
@@ -21,6 +21,8 @@ source "$(dirname "$0")"/common.sh
 
 DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker
 DOCKER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts
+DOCKER_IMAGE_BUILD_RETRIES=3
+BUILD_BACKOFF_TIME=5
 
 export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount
 export FLINK_DOCKER_IMAGE_NAME=test_docker_embedded_job
@@ -30,12 +32,19 @@ export INPUT_PATH=/data/test/input
 export OUTPUT_PATH=/data/test/output
 export FLINK_JOB_ARGUMENTS="--input ${INPUT_PATH}/words --output ${OUTPUT_PATH}/docker_wc_out"
 
-# user inside the container must be able to createto workaround in-container permissions
+build_image() {
+ ./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
+}
+
+# user inside the container must be able to create files, this is a workaround in-container permissions
 mkdir -p $OUTPUT_VOLUME
 chmod 777 $OUTPUT_VOLUME
 
 cd "$DOCKER_MODULE_DIR"
-./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
+if ! retry_times $DOCKER_IMAGE_BUILD_RETRIES ${BUILD_BACKOFF_TIME} build_image; then
+ echo "Failed to build docker image. Aborting..."
+ exit 1
+fi
 cd "$END_TO_END_DIR"
 
 docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --abort-on-container-exit --exit-code-from job-cluster &> /dev/null