From d95d395d92acca210e107663ed0e96c3285f0cf5 Mon Sep 17 00:00:00 2001 From: Wei Zhong Date: Fri, 10 May 2019 09:36:27 +0800 Subject: [PATCH] [FLINK-12370][python][travis] Integrated Travis for Python API. Brief change log: - Added python stage for Python API CI testing. - Integrated flink-python/dev/lint-python.sh for CI testing of the Python API. This closes #8392 --- .travis.yml | 17 ++++ tools/travis/stage.sh | 1 + tools/travis_controller.sh | 4 +- ...vis_mvn_watchdog.sh => travis_watchdog.sh} | 89 ++++++++++++------- 4 files changed, 75 insertions(+), 36 deletions(-) rename tools/{travis_mvn_watchdog.sh => travis_watchdog.sh} (77%) diff --git a/.travis.yml b/.travis.yml index f8243b46463c6..3f1583536447b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,6 +83,10 @@ jobs: script: ./tools/travis_controller.sh core env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" name: core + - if: type in (pull_request, push) + script: ./tools/travis_controller.sh python + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" + name: python - if: type in (pull_request, push) script: ./tools/travis_controller.sh libraries env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11" @@ -115,6 +119,10 @@ jobs: script: ./tools/travis_controller.sh core env: PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis" name: core - hadoop 2.4.1 + - if: type = cron + script: ./tools/travis_controller.sh python + env: PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis" + name: python - hadoop 2.4.1 - if: type = cron script: ./tools/travis_controller.sh libraries env: PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis" @@ -147,6 +155,10 @@ jobs: script: ./tools/travis_controller.sh core env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.12" name: core - scala 2.12 + - if: type = cron + script: ./tools/travis_controller.sh python + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.12" + name: python - scala 2.12 - if: type = cron script: ./tools/travis_controller.sh libraries env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.12" @@ -181,6 +193,11 @@ jobs: script: ./tools/travis_controller.sh core env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11 -Djdk9" name: core - jdk 9 + - if: type = cron + jdk: "openjdk9" + script: ./tools/travis_controller.sh python + env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11 -Djdk9" + name: python - if: type = cron jdk: "openjdk9" script: ./tools/travis_controller.sh libraries diff --git a/tools/travis/stage.sh b/tools/travis/stage.sh index 8fa43677cf823..3e84b2a10738f 100644 --- a/tools/travis/stage.sh +++ b/tools/travis/stage.sh @@ -19,6 +19,7 @@ STAGE_COMPILE="compile" STAGE_CORE="core" +STAGE_PYTHON="python" STAGE_LIBRARIES="libraries" STAGE_CONNECTORS="connectors" STAGE_TESTS="tests" diff --git a/tools/travis_controller.sh b/tools/travis_controller.sh index d659143ae8dfb..eef6503c7b646 100755 --- a/tools/travis_controller.sh +++ b/tools/travis_controller.sh @@ -158,7 +158,7 @@ if [ $STAGE == "$STAGE_COMPILE" ]; then # by removing files not required for subsequent stages # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided) - find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' | xargs rm -rf + find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' ! -path "$CACHE_FLINK_DIR/flink-dist/*" ! -path "*tests.jar" | xargs rm -rf # .git directory # not deleting this can cause build stability issues @@ -201,7 +201,7 @@ elif [ $STAGE != "$STAGE_CLEANUP" ]; then travis_time_finish end_fold "adjust_timestamps" - TEST="$STAGE" "./tools/travis_mvn_watchdog.sh" 300 + TEST="$STAGE" "./tools/travis_watchdog.sh" 300 EXIT_CODE=$? elif [ $STAGE == "$STAGE_CLEANUP" ]; then echo "Cleaning up $CACHE_BUILD_DIR" diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_watchdog.sh similarity index 77% rename from tools/travis_mvn_watchdog.sh rename to tools/travis_watchdog.sh index 4dff95f6a9a84..d53526f4f6b6e 100755 --- a/tools/travis_mvn_watchdog.sh +++ b/tools/travis_watchdog.sh @@ -45,6 +45,11 @@ SLEEP_TIME=20 LOG4J_PROPERTIES=${HERE}/log4j-travis.properties +PYTHON_TEST="./flink-python/dev/lint-python.sh" +PYTHON_PID="${ARTIFACTS_DIR}/watchdog.python.pid" +PYTHON_EXIT="${ARTIFACTS_DIR}/watchdog.python.exit" +PYTHON_OUT="${ARTIFACTS_DIR}/python.out" + MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${TEST}) MVN_TEST_MODULES=$(get_test_modules_for_stage ${TEST}) @@ -78,6 +83,20 @@ UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz +if [ $TEST == $STAGE_PYTHON ]; then + CMD=$PYTHON_TEST + CMD_PID=$PYTHON_PID + CMD_OUT=$PYTHON_OUT + CMD_EXIT=$PYTHON_EXIT + CMD_TYPE="PYTHON" +else + CMD=$MVN_COMPILE + CMD_PID=$MVN_PID + CMD_OUT=$MVN_OUT + CMD_EXIT=$MVN_EXIT + CMD_TYPE="MVN" +fi + # ============================================================================= # FUNCTIONS # ============================================================================= @@ -145,10 +164,10 @@ put_yarn_logs_to_artifacts() { mod_time () { if [[ `uname` == 'Darwin' ]]; then - eval $(stat -s $MVN_OUT) + eval $(stat -s $CMD_OUT) echo $st_mtime else - echo `stat -c "%Y" $MVN_OUT` + echo `stat -c "%Y" $CMD_OUT` fi } @@ -157,7 +176,7 @@ the_time() { } watchdog () { - touch $MVN_OUT + touch $CMD_OUT while true; do sleep $SLEEP_TIME @@ -171,7 +190,7 @@ watchdog () { print_stacktraces | tee $TRACE_OUT - kill $(<$MVN_PID) + kill $(<$CMD_PID) exit 1 fi @@ -194,54 +213,56 @@ cd $HERE/../ # Compile modules -echo "RUNNING '${MVN_COMPILE}'." +echo "RUNNING '${CMD}'." -# Run $MVN_COMPILE and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to -# allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains +# Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to +# allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains # the exit code. This is important for Travis' build life-cycle (success/failure). -( $MVN_COMPILE & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT +( $CMD & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT -EXIT_CODE=$(<$MVN_EXIT) +EXIT_CODE=$(<$CMD_EXIT) -echo "MVN exited with EXIT CODE: ${EXIT_CODE}." +echo "${CMD_TYPE} exited with EXIT CODE: ${EXIT_CODE}." -# Make sure to kill the watchdog in any case after $MVN_COMPILE has completed +# Make sure to kill the watchdog in any case after $CMD has completed echo "Trying to KILL watchdog (${WD_PID})." ( kill $WD_PID 2>&1 ) > /dev/null -rm $MVN_PID -rm $MVN_EXIT +rm $CMD_PID +rm $CMD_EXIT # Run tests if compilation was successful -if [ $EXIT_CODE == 0 ]; then +if [ $CMD_TYPE == "MVN" ]; then + if [ $EXIT_CODE == 0 ]; then - # Start watching $MVN_OUT - watchdog & - echo "STARTED watchdog (${WD_PID})." + # Start watching $MVN_OUT + watchdog & + echo "STARTED watchdog (${WD_PID})." - WD_PID=$! + WD_PID=$! - echo "RUNNING '${MVN_TEST}'." + echo "RUNNING '${MVN_TEST}'." - # Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to - # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains - # the exit code. This is important for Travis' build life-cycle (success/failure). - ( $MVN_TEST & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT + # Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to + # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains + # the exit code. This is important for Travis' build life-cycle (success/failure). + ( $MVN_TEST & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT - EXIT_CODE=$(<$MVN_EXIT) + EXIT_CODE=$(<$MVN_EXIT) - echo "MVN exited with EXIT CODE: ${EXIT_CODE}." + echo "MVN exited with EXIT CODE: ${EXIT_CODE}." - # Make sure to kill the watchdog in any case after $MVN_TEST has completed - echo "Trying to KILL watchdog (${WD_PID})." - ( kill $WD_PID 2>&1 ) > /dev/null + # Make sure to kill the watchdog in any case after $MVN_TEST has completed + echo "Trying to KILL watchdog (${WD_PID})." + ( kill $WD_PID 2>&1 ) > /dev/null - rm $MVN_PID - rm $MVN_EXIT -else - echo "==============================================================================" - echo "Compilation failure detected, skipping test execution." - echo "==============================================================================" + rm $MVN_PID + rm $MVN_EXIT + else + echo "==============================================================================" + echo "Compilation failure detected, skipping test execution." + echo "==============================================================================" + fi fi # Post