[FLINK-14382][e2e] Update e2e tests to support two inputs from two du…

…mmy FS.

flink-end-to-end-tests/test-scripts/common_dummy_fs.sh

@@ -19,5 +19,7 @@
 
 function dummy_fs_setup() {
  mkdir -p "$FLINK_DIR/plugins/dummy-fs"
- cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/plugins/dummy-fs/"
+ mkdir -p "$FLINK_DIR/plugins/another-dummy-fs"
+ cp "${END_TO_END_DIR}/flink-plugins-test/dummy-fs/target/flink-dummy-fs.jar" "${FLINK_DIR}/plugins/dummy-fs/"
+ cp "${END_TO_END_DIR}/flink-plugins-test/another-dummy-fs/target/flink-another-dummy-fs.jar" "${FLINK_DIR}/plugins/another-dummy-fs/"
 }

flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh

@@ -20,29 +20,31 @@
 source "$(dirname "$0")"/common.sh
 
 INPUT_TYPE=${1:-file}
+RESULT_HASH="72a690412be8928ba239c2da967328a5"
 case $INPUT_TYPE in
  (file)
- INPUT_LOCATION="${TEST_INFRA_DIR}/test-data/words"
+ INPUT_ARGS="--input ${TEST_INFRA_DIR}/test-data/words"
  ;;
  (hadoop)
  source "$(dirname "$0")"/common_s3.sh
  s3_setup hadoop
- INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
+ INPUT_ARGS="--input ${S3_TEST_DATA_WORDS_URI}"
  ;;
  (hadoop_with_provider)
  source "$(dirname "$0")"/common_s3.sh
  s3_setup_with_provider hadoop "fs.s3a.aws.credentials.provider"
- INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
+ INPUT_ARGS="--input ${S3_TEST_DATA_WORDS_URI}"
  ;;
  (presto)
  source "$(dirname "$0")"/common_s3.sh
  s3_setup presto
- INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
+ INPUT_ARGS="--input ${S3_TEST_DATA_WORDS_URI}"
  ;;
  (dummy-fs)
  source "$(dirname "$0")"/common_dummy_fs.sh
  dummy_fs_setup
- INPUT_LOCATION="dummy:https://localhost/words"
+ INPUT_ARGS="--input dummy:https://localhost/words --input anotherDummy:https://localhost/words"
+ RESULT_HASH="0e5bd0a3dd7d5a7110aa85ff70adb54b"
  ;;
  (*)
  echo "Unknown input type $INPUT_TYPE"
@@ -58,5 +60,5 @@ start_cluster
 
 # The test may run against different source types.
 # But the sources should provide the same test data, so the checksum stays the same for all tests.
-"${FLINK_DIR}/bin/flink" run -p 1 "${FLINK_DIR}/examples/batch/WordCount.jar" --input "${INPUT_LOCATION}" --output "${OUTPUT_LOCATION}"
-check_result_hash "WordCount (${INPUT_TYPE})" "${OUTPUT_LOCATION}" "72a690412be8928ba239c2da967328a5"
+eval "${FLINK_DIR}/bin/flink run -p 1 ${FLINK_DIR}/examples/batch/WordCount.jar ${INPUT_ARGS} --output ${OUTPUT_LOCATION}"
+check_result_hash "WordCount (${INPUT_TYPE})" "${OUTPUT_LOCATION}" "${RESULT_HASH}"

flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh

@@ -32,22 +32,24 @@ export INPUT_PATH=/data/test/input
 export OUTPUT_PATH=/data/test/output
 
 INPUT_TYPE=${1:-file}
+RESULT_HASH="72a690412be8928ba239c2da967328a5"
 case $INPUT_TYPE in
  (file)
- INPUT_LOCATION=${INPUT_PATH}/words
+ INPUT_ARGS="--input ${INPUT_PATH}/words"
  ;;
  (dummy-fs)
  source "$(dirname "$0")"/common_dummy_fs.sh
  dummy_fs_setup
- INPUT_LOCATION="dummy:https://localhost/words"
+ INPUT_ARGS="--input dummy:https://localhost/words --input anotherDummy:https://localhost/words"
+ RESULT_HASH="0e5bd0a3dd7d5a7110aa85ff70adb54b"
  ;;
  (*)
  echo "Unknown input type $INPUT_TYPE"
  exit 1
  ;;
 esac
 
-export FLINK_JOB_ARGUMENTS="--input ${INPUT_LOCATION} --output ${OUTPUT_PATH}/docker_wc_out"
+export FLINK_JOB_ARGUMENTS="${INPUT_ARGS} --output ${OUTPUT_PATH}/docker_wc_out"
 
 build_image() {
  ./build.sh --from-local-dist --job-artifacts ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
@@ -68,4 +70,4 @@ docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/d
 docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs job-cluster > ${FLINK_DIR}/log/jobmanager.log
 docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs taskmanager > ${FLINK_DIR}/log/taskmanager.log
 
-check_result_hash "WordCount" $OUTPUT_VOLUME/docker_wc_out "72a690412be8928ba239c2da967328a5"
+check_result_hash "WordCount" $OUTPUT_VOLUME/docker_wc_out "${RESULT_HASH}"

flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh

@@ -31,8 +31,8 @@ case $INPUT_TYPE in
  (dummy-fs)
  source "$(dirname "$0")"/common_dummy_fs.sh
  dummy_fs_setup
- INPUT_ARGS="--input dummy:https://localhost/words"
- EXPECTED_RESULT_LOG_CONTAINS=("my,1" "dear,2" "world,2")
+ INPUT_ARGS="--input dummy:https://localhost/words --input anotherDummy:https://localhost/words"
+ EXPECTED_RESULT_LOG_CONTAINS=("my,2" "dear,4" "world,4")
  ;;
  (*)
  echo "Unknown input type $INPUT_TYPE"