[FLINK-20845] Drop Scala 2.11 support

mbalassi · Sep 30, 2021 · d80d487 · d80d487
1 parent 6b58562
commit d80d487
Show file tree

Hide file tree

Showing 27 changed files with 49 additions and 175 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -76,7 +76,7 @@ stages:
  vmImage: 'ubuntu-20.04'
  e2e_pool_definition:
  vmImage: 'ubuntu-20.04'
- environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+ environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12"
  run_end_to_end: false
  container: flink-build-container
  jdk: 8
@@ -93,5 +93,5 @@ stages:
  - template: tools/azure-pipelines/build-python-wheels.yml
  parameters:
  stage_name: cron_python_wheels
- environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+ environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12"
  container: flink-build-container
diff --git a/docs/README.md b/docs/README.md
@@ -110,7 +110,7 @@ This will be replaced by the maven artifact for flink-streaming-java that users
 ```xml
 <dependency>
  <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.11</artifactId>
+ <artifactId>flink-streaming-java_2.12</artifactId>
  <version><!-- current flink version --></version>
 </dependency>
 ```

diff --git a/docs/config.toml b/docs/config.toml
@@ -52,7 +52,7 @@ pygmentsUseClasses = true
  TrainingExercises = "//github.com/apache/flink-training"
 
  # This suffix is appended to the Scala-dependent Maven artifact names
- ScalaVersion = "_2.11"
+ ScalaVersion = "_2.12"
 
  ProjectHomepage = "//flink.apache.org"
 

diff --git a/docs/content.zh/docs/deployment/resource-providers/standalone/docker.md b/docs/content.zh/docs/deployment/resource-providers/standalone/docker.md
@@ -232,7 +232,7 @@ Images for each supported combination of Flink and Scala versions are available,
 For example, you can use the following aliases:
 
 * `flink:latest` → `flink:<latest-flink>-scala_<latest-scala>`
-* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.11`
+* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.12`
 
 <span class="label label-info">Note</span> It is recommended to always use an explicit version tag of the docker image that specifies both the needed Flink and Scala
 versions (for example `flink:1.11-scala_2.12`).

diff --git a/docs/content.zh/docs/dev/datastream/project-configuration.md b/docs/content.zh/docs/dev/datastream/project-configuration.md
@@ -151,7 +151,7 @@ For that reason, Flink for Scala 2.11 cannot be used with an application that us
 Scala 2.12.
 
 All Flink dependencies that (transitively) depend on Scala are suffixed with the
-Scala version that they are built for, for example `flink-streaming-scala_2.11`.
+Scala version that they are built for, for example `flink-streaming-scala_2.12`.
 
 Developers that only use Java can pick any Scala version, Scala developers need to
 pick the Scala version that matches their application's Scala version.

diff --git a/docs/content/docs/deployment/resource-providers/standalone/docker.md b/docs/content/docs/deployment/resource-providers/standalone/docker.md
@@ -232,7 +232,7 @@ Images for each supported combination of Flink and Scala versions are available,
 For example, you can use the following aliases:
 
 * `flink:latest` → `flink:<latest-flink>-scala_<latest-scala>`
-* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.11`
+* `flink:1.11` → `flink:1.11.<latest-flink-1.11>-scala_2.12`
 
 <span class="label label-info">Note</span> It is recommended to always use an explicit version tag of the docker image that specifies both the needed Flink and Scala
 versions (for example `flink:1.11-scala_2.12`).

diff --git a/docs/content/docs/dev/datastream/project-configuration.md b/docs/content/docs/dev/datastream/project-configuration.md
@@ -150,7 +150,7 @@ For that reason, Flink for Scala 2.11 cannot be used with an application that us
 Scala 2.12.
 
 All Flink dependencies that (transitively) depend on Scala are suffixed with the
-Scala version that they are built for, for example `flink-streaming-scala_2.11`.
+Scala version that they are built for, for example `flink-streaming-scala_2.12`.
 
 Developers that only use Java can pick any Scala version, Scala developers need to
 pick the Scala version that matches their application's Scala version.

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
@@ -424,15 +424,6 @@ under the License.
  </dependencies>
 
  <profiles>
- <profile>
- <id>scala-2.11</id>
- <activation>
- <property>
- <name>!scala-2.12</name>
- </property>
- </activation>
- </profile>
-
  <profile>
  <!-- Creates/Removes the 'build-target' symlink in the root directory (only Unix systems) -->
  <id>symlink-build-target</id>

diff --git a/flink-dist/src/main/resources/META-INF/NOTICE b/flink-dist/src/main/resources/META-INF/NOTICE
@@ -9,7 +9,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.ververica:frocksdbjni:6.20.3-ververica-1.0
 - com.google.code.findbugs:jsr305:1.3.9
 - com.twitter:chill-java:0.7.6
-- com.twitter:chill_2.11:0.7.6
+- com.twitter:chill_2.12:0.7.6
 - commons-cli:commons-cli:1.4
 - commons-collections:commons-collections:3.2.2
 - commons-io:commons-io:2.11.0
@@ -29,11 +29,10 @@ See bundled license files for details.
 
 The following dependencies all share the same BSD license which you find under licenses/LICENSE.scala.
 
-- org.scala-lang:scala-compiler:2.11.12
-- org.scala-lang:scala-library:2.11.12
-- org.scala-lang:scala-reflect:2.11.12
-- org.scala-lang.modules:scala-parser-combinators_2.11:1.1.1
-- org.scala-lang.modules:scala-xml_2.11:1.0.5
+- org.scala-lang:scala-compiler:2.12.7
+- org.scala-lang:scala-library:2.12.7
+- org.scala-lang:scala-reflect:2.12.7
+- org.scala-lang.modules:scala-xml_2.12:1.0.6
 
 This project bundles the following dependencies under the MIT/X11 license.
 See bundled license files for details.

diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
@@ -88,6 +88,8 @@ under the License.
  <module>flink-python-test</module>
  <module>flink-end-to-end-tests-hbase</module>
  <module>flink-end-to-end-tests-pulsar</module>
+ <module>flink-glue-schema-registry-avro-test</module>
+ <module>flink-glue-schema-registry-json-test</module>
  </modules>
 
  <dependencyManagement>
@@ -171,23 +173,6 @@ under the License.
  <e2e.exclude.org.apache.flink.testutils.junit.FailsOnJava11>true</e2e.exclude.org.apache.flink.testutils.junit.FailsOnJava11>
  </properties>
  </profile>
-
- <profile>
- <id>scala-2.12</id>
- <properties>
- <scala.version>2.12.7</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
- </properties>
- <activation>
- <property>
- <name>scala-2.12</name>
- </property>
- </activation>
- <modules>
- <module>flink-glue-schema-registry-avro-test</module>
- <module>flink-glue-schema-registry-json-test</module>
- </modules>
- </profile>
  </profiles>
 
  <build>

diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -253,11 +253,7 @@ LOG4J_PROPERTIES=${END_TO_END_DIR}/../tools/ci/log4j.properties
 
 MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -DlogBackupDir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file:https://$LOG4J_PROPERTIES"
 MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build"
-if [[ ${PROFILE} != *"scala-2.12"* ]]; then
- e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -not -path '*glue-schema-registry*' -printf '%h\n' | sort -u | tr '\n' ',')
-else
- e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
-fi
+e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
 e2e_modules="${e2e_modules},$(find flink-walkthroughs -mindepth 2 -maxdepth 2 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')"
 
 PROFILE="$PROFILE -Pe2e-travis1 -Pe2e-travis2 -Pe2e-travis3 -Pe2e-travis4 -Pe2e-travis5 -Pe2e-travis6"

diff --git a/flink-end-to-end-tests/test-scripts/test_quickstarts.sh b/flink-end-to-end-tests/test-scripts/test_quickstarts.sh
@@ -70,12 +70,6 @@ sed -i -e "s/org.apache.flink.quickstart.StreamingJob/org.apache.flink.quickstar
 
 case $PROFILE in
 *"scala-2.12"*)
- echo "Changing scala version"
- sed -i -e "s/scala.binary.version>2.11<\/scala.binary/scala.binary.version>2.12<\/scala.binary/" pom.xml
- # for flink-quickstart-scala, also change scala version
- sed -i -e "s/scala.version>2.11.12<\/scala.ver/scala.version>2.12.7<\/scala.ver/" pom.xml
- ;;
-*"scala-2.11"*)
  # all good
  ;;
 *"scala-"*)

diff --git a/...flink-avro/src/test/java/org/apache/flink/formats/avro/AvroSplittableInputFormatTest.java b/...flink-avro/src/test/java/org/apache/flink/formats/avro/AvroSplittableInputFormatTest.java
@@ -353,7 +353,7 @@ public void testAvroRecovery() throws Exception {
 
  <dependency>
  <groupId>org.apache.flink</groupId>
- <artifactId>flink-hadoop-compatibility_2.11</artifactId>
+ <artifactId>flink-hadoop-compatibility_2.12</artifactId>
  <version>1.6-SNAPSHOT</version>
  </dependency>
 

diff --git a/flink-formats/pom.xml b/flink-formats/pom.xml
@@ -50,6 +50,8 @@ under the License.
  <module>flink-orc-nohive</module>
  <module>flink-hadoop-bulk</module>
  <module>flink-format-common</module>
+ <module>flink-avro-glue-schema-registry</module>
+ <module>flink-json-glue-schema-registry</module>
  </modules>
 
  <!-- override these root dependencies as 'provided', so they don't end up
@@ -86,23 +88,6 @@ under the License.
  <module>flink-sql-avro-confluent-registry</module>
  </modules>
  </profile>
-
- <profile>
- <id>scala-2.12</id>
- <properties>
- <scala.version>2.12.7</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
- </properties>
- <activation>
- <property>
- <name>scala-2.12</name>
- </property>
- </activation>
- <modules>
- <module>flink-avro-glue-schema-registry</module>
- <module>flink-json-glue-schema-registry</module>
- </modules>
- </profile>
  </profiles>
 
 </project>
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -31,7 +31,7 @@ under the License.
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  <flink.version>@project.version@</flink.version>
  <target.java.version>@target.java.version@</target.java.version>
- <scala.binary.version>2.11</scala.binary.version>
+ <scala.binary.version>2.12</scala.binary.version>
  <maven.compiler.source>${target.java.version}</maven.compiler.source>
  <maven.compiler.target>${target.java.version}</maven.compiler.target>
  <log4j.version>@log4j.version@</log4j.version>

diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -46,8 +46,8 @@ under the License.
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  <flink.version>@project.version@</flink.version>
  <target.java.version>@target.java.version@</target.java.version>
- <scala.binary.version>2.11</scala.binary.version>
- <scala.version>2.11.12</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
+ <scala.version>2.12.7</scala.version>
  <log4j.version>@log4j.version@</log4j.version>
  </properties>
 

diff --git a/flink-rpc/flink-rpc-akka/pom.xml b/flink-rpc/flink-rpc-akka/pom.xml
@@ -152,6 +152,11 @@ under the License.
  <artifactId>scala-reflect</artifactId>
  <version>${akka.scala.version}</version>
  </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
+ <version>1.1.2</version>
+ </dependency>
  </dependencies>
  </dependencyManagement>
 

diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/ClosureCleaner.scala
@@ -39,8 +39,6 @@ object ClosureCleaner {
 
  val LOG = LoggerFactory.getLogger(this.getClass)
 
- private val isScala2_11 = scala.util.Properties.versionString.contains("2.11")
-
  // Get an ASM class reader for a given class from the JAR that loaded it
  private[scala] def getClassReader(cls: Class[_]): ClassReader = {
  // Copy data over, before delegating to ClassReader - else we can run out of open file handles.
@@ -174,9 +172,6 @@ object ClosureCleaner {
  * @param closure the closure to check.
  */
  private def getSerializedLambda(closure: AnyRef): Option[SerializedLambda] = {
- if (isScala2_11) {
- return None
- }
  val isClosureCandidate =
  closure.getClass.isSynthetic &&
  closure

diff --git a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
@@ -16,7 +16,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.calcite:calcite-core:1.26.0
 - org.apache.calcite:calcite-linq4j:1.26.0
 - org.apache.calcite.avatica:avatica-core:1.17.0
-- org.scala-lang.modules:scala-parser-combinators_2.11:1.1.1
+- org.scala-lang.modules:scala-parser-combinators_2.12:1.1.1
 - commons-codec:commons-codec:1.15
 - commons-io:commons-io:2.11.0
 

diff --git a/...planner/src/main/scala/org/apache/flink/table/planner/utils/JavaScalaConversionUtil.scala b/...planner/src/main/scala/org/apache/flink/table/planner/utils/JavaScalaConversionUtil.scala
@@ -30,42 +30,28 @@ import scala.collection.JavaConverters._
  */
 object JavaScalaConversionUtil {
 
- // most of these methods are not necessary once we upgraded to Scala 2.12
-
  def toJava[T](option: Option[T]): Optional[T] = option match {
  case Some(v) => Optional.of(v)
  case None => Optional.empty()
  }
 
  def toScala[T](option: Optional[T]): Option[T] = Option(option.orElse(null.asInstanceOf[T]))
 
- def toJava[T](func: (T) => Unit): Consumer[T] = new Consumer[T] {
- override def accept(t: T): Unit = {
- func.apply(t)
- }
- }
+ def toJava[T](func: T => Unit): Consumer[T] =
+ func(_)
 
- def toJava[K, V](func: (K, V) => Unit): BiConsumer[K, V] = new BiConsumer[K, V] {
- override def accept(k: K, v: V): Unit = {
- func.apply(k ,v)
- }
- }
+ def toJava[K, V](func: (K, V) => Unit): BiConsumer[K, V] =
+ func(_, _)
 
- def toJava[I, O](func: (I) => O): Function[I, O] = new Function[I, O] {
- override def apply(in: I): O = {
- func.apply(in)
- }
- }
+ def toJava[I, O](func: I => O): Function[I, O] =
+ func(_)
 
- def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = {
+ def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] =
  new JTuple2[T0, T1](tuple._1, tuple._2)
- }
 
- def toJava[T](seq: Seq[T]): JList[T] = {
+ def toJava[T](seq: Seq[T]): JList[T] =
  seq.asJava
- }
 
- def toScala[T](list: JList[T]): Seq[T] = {
+ def toScala[T](list: JList[T]): Seq[T] =
  list.asScala
- }
 }
diff --git a/...throughs/flink-walkthrough-datastream-java/src/main/resources/archetype-resources/pom.xml b/...throughs/flink-walkthrough-datastream-java/src/main/resources/archetype-resources/pom.xml
@@ -32,7 +32,7 @@ under the License.
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  <flink.version>@project.version@</flink.version>
  <target.java.version>1.8</target.java.version>
- <scala.binary.version>2.11</scala.binary.version>
+ <scala.binary.version>2.12</scala.binary.version>
  <maven.compiler.source>${target.java.version}</maven.compiler.source>
  <maven.compiler.target>${target.java.version}</maven.compiler.target>
  <log4j.version>@log4j.version@</log4j.version>

diff --git a/...hroughs/flink-walkthrough-datastream-scala/src/main/resources/archetype-resources/pom.xml b/...hroughs/flink-walkthrough-datastream-scala/src/main/resources/archetype-resources/pom.xml
@@ -46,8 +46,8 @@ under the License.
  <properties>
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  <flink.version>@project.version@</flink.version>
- <scala.binary.version>2.11</scala.binary.version>
- <scala.version>2.11.12</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
+ <scala.version>2.12.7</scala.version>
  <log4j.version>@log4j.version@</log4j.version>
  </properties>