[FLINK-11100][s3][tests] Add FS type argument to s3_setup

deepfield · Dec 18, 2018 · 030743f · 030743f
1 parent cf88590
commit 030743f
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 4 deletions.
diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh
@@ -58,7 +58,7 @@ s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.ja
 # IT_CASE_S3_ACCESS_KEY
 # IT_CASE_S3_SECRET_KEY
 # Arguments:
-# None
+# $1 - s3 filesystem type (hadoop/presto)
 # Returns:
 # None
 ###################################
@@ -73,13 +73,11 @@ function s3_setup {
  }
  trap s3_cleanup EXIT
 
- cp $FLINK_DIR/opt/flink-s3-fs-hadoop-*.jar $FLINK_DIR/lib/
+ cp $FLINK_DIR/opt/flink-s3-fs-$1-*.jar $FLINK_DIR/lib/
  echo "s3.access-key: $IT_CASE_S3_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
  echo "s3.secret-key: $IT_CASE_S3_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
 }
 
-s3_setup
-
 ###################################
 # List s3 objects by full path prefix.
 #

diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
@@ -22,6 +22,7 @@
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/common_s3.sh
 
+s3_setup hadoop
 start_cluster
 
 $FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out

diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
@@ -22,6 +22,7 @@
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/common_s3.sh
 
+s3_setup presto
 start_cluster
 
 $FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out

diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
@@ -22,6 +22,7 @@ OUT_TYPE="${1:-local}" # other type: s3
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/common_s3.sh
 
+s3_setup hadoop
 set_conf_ssl "mutual"
 
 OUT=temp/test_streaming_file_sink-$(uuidgen)