[BEAM-7962] Drop support for Flink 1.5 and 1.6

Flink 1.9 is now released and Beam 2.17.0 is going to support it. Since the
Flink community only supports the last two Flink releases, it is now time to
drop at least 1.5 and 1.6.

As a follow-up, we will get rid of Flink 1.5/1.6 specific workarounds, e.g. make use of Flink's preSnapshotBarrier in AbstractStreamOperator which removes the needs to buffer elements during a snapshot.

Build time should decrease by several minutes.

.test-infra/jenkins/CommonTestProperties.groovy

@@ -35,7 +35,7 @@ class CommonTestProperties {
  JAVA: [
  DATAFLOW: ":runners:google-cloud-dataflow-java",
  SPARK: ":runners:spark",
- FLINK: ":runners:flink:1.5",
+ FLINK: ":runners:flink:1.8",
  DIRECT: ":runners:direct-java"
  ],
  PYTHON: [

.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy

@@ -40,7 +40,7 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
  rootBuildScriptDir(commonJobProperties.checkoutDir)
  tasks(':sdks:java:testing:nexmark:run')
  commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.5"' +
+ switches('-Pnexmark.runner=":runners:flink:1.8"' +
  ' -Pnexmark.args="' +
  [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
  '--streaming=false',
@@ -55,7 +55,7 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
  rootBuildScriptDir(commonJobProperties.checkoutDir)
  tasks(':sdks:java:testing:nexmark:run')
  commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.5"' +
+ switches('-Pnexmark.runner=":runners:flink:1.8"' +
  ' -Pnexmark.args="' +
  [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
  '--streaming=true',
@@ -70,7 +70,7 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
  rootBuildScriptDir(commonJobProperties.checkoutDir)
  tasks(':sdks:java:testing:nexmark:run')
  commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.5"' +
+ switches('-Pnexmark.runner=":runners:flink:1.8"' +
  ' -Pnexmark.args="' +
  [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
  '--queryLanguage=sql',
@@ -85,7 +85,7 @@ NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
  rootBuildScriptDir(commonJobProperties.checkoutDir)
  tasks(':sdks:java:testing:nexmark:run')
  commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.5"' +
+ switches('-Pnexmark.runner=":runners:flink:1.8"' +
  ' -Pnexmark.args="' +
  [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
  '--queryLanguage=sql',

.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy

@@ -36,7 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Batch',
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.5:job-server:validatesPortableRunnerBatch')
+ tasks(':runners:flink:1.8:job-server:validatesPortableRunnerBatch')
  commonJobProperties.setGradleSwitches(delegate)
  }
  }

.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy

@@ -36,7 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Streaming',
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.5:job-server:validatesPortableRunnerStreaming')
+ tasks(':runners:flink:1.8:job-server:validatesPortableRunnerStreaming')
  commonJobProperties.setGradleSwitches(delegate)
  }
  }

.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy

@@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Flink',
  steps {
  gradle {
  rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.5:validatesRunner')
+ tasks(':runners:flink:1.8:validatesRunner')
  commonJobProperties.setGradleSwitches(delegate)
  }
  }

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -1336,7 +1336,7 @@ class BeamModulePlugin implements Plugin<Project> {
  }
 
  if (runner?.equalsIgnoreCase('flink')) {
- testRuntime it.project(path: ":runners:flink:1.5", configuration: 'testRuntime')
+ testRuntime it.project(path: ":runners:flink:1.8", configuration: 'testRuntime')
  }
 
  if (runner?.equalsIgnoreCase('spark')) {
@@ -1808,7 +1808,7 @@ class BeamModulePlugin implements Plugin<Project> {
  dependsOn setupTask
  // We need flink-job-server-container dependency since Python PortableRunner automatically
  // brings the flink-job-server-container up when --job_endpoint is not specified.
- dependsOn ':runners:flink:1.5:job-server-container:docker'
+ dependsOn ':runners:flink:1.8:job-server-container:docker'
  }
  mainTask.dependsOn pythonTask
  cleanupTask.mustRunAfter pythonTask
@@ -1990,7 +1990,7 @@ class BeamModulePlugin implements Plugin<Project> {
  project.task('portableWordCount' + (isStreaming ? 'Streaming' : 'Batch')) {
  dependsOn = ['installGcpTest']
  mustRunAfter = [
- ':runners:flink:1.5:job-server-container:docker',
+ ':runners:flink:1.8:job-server-container:docker',
  ':sdks:python:container:py2:docker',
  ':sdks:python:container:py35:docker',
  ':sdks:python:container:py36:docker',

examples/java/build.gradle

@@ -78,7 +78,7 @@ dependencies {
  // https://issues.apache.org/jira/browse/BEAM-3583
  // apexRunnerPreCommit project(":runners:apex")
  directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")
- flinkRunnerPreCommit project(":runners:flink:1.5")
+ flinkRunnerPreCommit project(":runners:flink:1.8")
  // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath
  // before 4.1.8.Final defined by Apache Beam
  sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final"

examples/kotlin/build.gradle

@@ -81,7 +81,7 @@ dependencies {
  // https://issues.apache.org/jira/browse/BEAM-3583
  // apexRunnerPreCommit project(":runners:apex")
  directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")
- flinkRunnerPreCommit project(":runners:flink:1.5")
+ flinkRunnerPreCommit project(":runners:flink:1.8")
  // TODO: Make the netty version used configurable, we add netty-all 4.1.17.Final so it appears on the classpath
  // before 4.1.8.Final defined by Apache Beam
  sparkRunnerPreCommit "io.netty:netty-all:4.1.17.Final"

project-mappings

@@ -94,9 +94,9 @@
 :beam-runners-google-cloud-dataflow-java-examples :runners:google-cloud-dataflow-java:examples
 :beam-runners-google-cloud-dataflow-java :runners:google-cloud-dataflow-java
 :beam-runners-gearpump :runners:gearpump
-:beam-runners-flink_2.11-job-server-container :runners:flink:1.5:job-server-container
-:beam-runners-flink_2.11-job-server :runners:flink:1.5:job-server
-:beam-runners-flink_2.11 :runners:flink:1.5
+:beam-runners-flink_2.11-job-server-container :runners:flink:1.8:job-server-container
+:beam-runners-flink_2.11-job-server :runners:flink:1.8:job-server
+:beam-runners-flink_2.11 :runners:flink:1.8
 :beam-runners-flink-1.7-job-server-container :runners:flink:1.7:job-server-container
 :beam-runners-flink-1.7-job-server :runners:flink:1.7:job-server
 :beam-runners-flink-1.7 :runners:flink:1.7

release/build.gradle

@@ -34,7 +34,7 @@ task runJavaExamplesValidationTask {
  dependsOn ":runners:google-cloud-dataflow-java:runQuickstartJavaDataflow"
  dependsOn ":runners:apex:runQuickstartJavaApex"
  dependsOn ":runners:spark:runQuickstartJavaSpark"
- dependsOn ":runners:flink:1.5:runQuickstartJavaFlinkLocal"
+ dependsOn ":runners:flink:1.8:runQuickstartJavaFlinkLocal"
  dependsOn ":runners:direct-java:runMobileGamingJavaDirect"
  dependsOn ":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow"
 }

release/src/main/scripts/run_rc_validation.sh

@@ -209,7 +209,7 @@ if [[ "$java_quickstart_flink_local" = true ]]; then
  echo "*************************************************************"
  echo "* Running Java Quickstart with Flink local runner"
  echo "*************************************************************"
- ./gradlew :runners:flink:1.5:runQuickstartJavaFlinkLocal \
+ ./gradlew :runners:flink:1.8:runQuickstartJavaFlinkLocal \
  -Prepourl=${REPO_URL} \
  -Pver=${RELEASE_VER}
 else

runners/flink/1.7/build.gradle

@@ -23,8 +23,8 @@ project.ext {
  // Set the version of all Flink-related dependencies here.
  flink_version = '1.7.2'
  // Main source directory and Flink version specific code.
- main_source_dirs = ["$basePath/src/main/java", "../1.5/src/main/java"]
- test_source_dirs = ["$basePath/src/test/java", "../1.5/src/test/java"]
+ main_source_dirs = ["$basePath/src/main/java", "./src/main/java"]
+ test_source_dirs = ["$basePath/src/test/java", "./src/test/java"]
  main_resources_dirs = ["$basePath/src/main/resources"]
  test_resources_dirs = ["$basePath/src/test/resources"]
  archives_base_name = 'beam-runners-flink-1.7'

runners/flink/flink_runner.gradle

@@ -83,14 +83,7 @@ test {
  }
  // TODO Running tests of all Flink versions in parallel can be too harsh on Jenkins memory
  // Run them serially for now, to avoid "Exit code 137", i.e. Jenkins host killing the Gradle test process
- if (project.path == ":runners:flink:1.6") {
- mustRunAfter(":runners:flink:1.5:test")
- } else if (project.path == ":runners:flink:1.7") {
- mustRunAfter(":runners:flink:1.5:test")
- mustRunAfter(":runners:flink:1.6:test")
- } else if (project.path == ":runners:flink:1.8") {
- mustRunAfter(":runners:flink:1.5:test")
- mustRunAfter(":runners:flink:1.6:test")
+ if (project.path == ":runners:flink:1.8") {
  mustRunAfter(":runners:flink:1.7:test")
  }
 }
@@ -190,5 +183,5 @@ task validatesRunner {
  dependsOn validatesRunnerStreaming
 }
 
-// Generates :runners:flink:1.5:runQuickstartJavaFlinkLocal
+// Generates :runners:flink:1.8:runQuickstartJavaFlinkLocal
 createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'FlinkLocal')

sdks/go/test/build.gradle

@@ -49,12 +49,12 @@ golang {
 
 task flinkValidatesRunner {
  dependsOn ":sdks:go:test:goBuild"
- dependsOn ":runners:flink:1.5:job-server:shadowJar"
+ dependsOn ":runners:flink:1.8:job-server:shadowJar"
  doLast {
  def options = [
  "--runner flink",
  "--parallel 1", // prevent memory overuse
- "--flink_job_server_jar ${project(":runners:flink:1.5:job-server").shadowJar.archivePath}",
+ "--flink_job_server_jar ${project(":runners:flink:1.8:job-server").shadowJar.archivePath}",
  ]
  exec {
  executable "sh"

sdks/java/testing/nexmark/build.gradle

@@ -102,7 +102,7 @@ if (shouldProvideSpark) {
 //
 // Parameters:
 // -Pnexmark.runner
-// Specify a runner subproject, such as ":runners:spark" or ":runners:flink:1.5"
+// Specify a runner subproject, such as ":runners:spark" or ":runners:flink:1.8"
 // Defaults to ":runners:direct-java"
 //
 // -Pnexmark.args