[FLINK-16330][AZP] Enable tests requiring S3 credentials

This closes apache#11332
derekm · Mar 12, 2020 · 222dc57 · 222dc57
1 parent 0523ef6
commit 222dc57
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -216,8 +216,6 @@ fi
 ARTIFACTS_DIR="${HERE}/artifacts"
 mkdir -p $ARTIFACTS_DIR || { echo "FAILURE: cannot create log directory '${ARTIFACTS_DIR}'." ; exit 1; }
 
-env > $ARTIFACTS_DIR/environment
-
 LOG4J_PROPERTIES=${HERE}/../tools/log4j-travis.properties
 
 MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file:https://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"

diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml
@@ -17,7 +17,7 @@
 #
 # This file defines the Flink build for the "apache/flink" repository, including
 # the following:
-# - PR builds
+# - PR builds (triggered through ci-bot)
 # - custom triggered e2e tests
 # - nightly builds
 

diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
@@ -122,6 +122,10 @@ jobs:
  # Test
  - script: STAGE=test ${{parameters.environment}} ./tools/azure_controller.sh $(module)
  displayName: Test - $(module)
+ env:
+ IT_CASE_S3_BUCKET: $(IT_CASE_S3_BUCKET)
+ IT_CASE_S3_ACCESS_KEY: $(IT_CASE_S3_ACCESS_KEY)
+ IT_CASE_S3_SECRET_KEY: $(IT_CASE_S3_SECRET_KEY)
 
  - task: PublishTestResults@2
  inputs:
@@ -162,13 +166,18 @@ jobs:
  displayName: Setup Docker
  - script: ./tools/azure-pipelines/free_disk_space.sh
  displayName: Free up disk space
+ - script: sudo apt-get install -y bc
  - script: M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH ${{parameters.environment}} STAGE=compile ./tools/azure_controller.sh compile
  displayName: Build Flink
  # TODO remove pre-commit tests script by adding the tests to the nightly script
 # - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh
 # displayName: Test - precommit 
  - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh
  displayName: Run e2e tests
+ env:
+ IT_CASE_S3_BUCKET: $(IT_CASE_S3_BUCKET)
+ IT_CASE_S3_ACCESS_KEY: $(IT_CASE_S3_ACCESS_KEY)
+ IT_CASE_S3_SECRET_KEY: $(IT_CASE_S3_SECRET_KEY)
  # upload debug artifacts
  - task: PublishPipelineArtifact@1
  condition: and(succeededOrFailed(), not(eq('$(ARTIFACT_DIR)', '')))