[FLINK-12024] Bump universal Kafka connector to Kafka dependency to 2…

….2.0 (apache#8055) This commit also removes useless arg:KAFKA_CONNECTOR_VERSION
bbejeck · Apr 25, 2019 · 40ebe25 · 40ebe25
1 parent 14c3d85
commit 40ebe25
Show file tree

Hide file tree

Showing 13 changed files with 26 additions and 25 deletions.
diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md
@@ -88,7 +88,7 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is
   <td>>= 1.0.0</td>
   <td>
   This universal Kafka connector attempts to track the latest version of the Kafka client.
-  The version of the client it uses may change between Flink releases.
+  The version of the client it uses may change between Flink releases. As of this release, it uses the Kafka 2.2.0 client.
   Modern Kafka clients are backwards compatible with broker versions 0.10.0 or later.
   However for Kafka 0.11.x and 0.10.x versions, we recommend using dedicated
   flink-connector-kafka-0.11{{ site.scala_version_suffix }} and flink-connector-kafka-0.10{{ site.scala_version_suffix }} respectively.

diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml
@@ -36,7 +36,7 @@ under the License.
  <packaging>jar</packaging>
 
  <properties>
- <kafka.version>2.0.1</kafka.version>
+ <kafka.version>2.2.0</kafka.version>
  </properties>
 
  <dependencies>

diff --git a/...java/org/apache/flink/streaming/connectors/kafka/internal/FlinkKafkaInternalProducer.java b/...java/org/apache/flink/streaming/connectors/kafka/internal/FlinkKafkaInternalProducer.java
@@ -45,6 +45,7 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -125,6 +126,11 @@ public void close(long timeout, TimeUnit unit) {
  kafkaProducer.close(timeout, unit);
  }
 
+ @Override
+ public void close(Duration duration) {
+ kafkaProducer.close(duration);
+ }
+
  // -------------------------------- New methods or methods with changed behaviour --------------------------------
 
  @Override

diff --git a/flink-connectors/flink-sql-connector-kafka/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-kafka/src/main/resources/META-INF/NOTICE
@@ -6,4 +6,4 @@ The Apache Software Foundation (http:https://www.apache.org/).
 
 This project bundles the following dependencies under the Apache Software License 2.0. (http:https://www.apache.org/licenses/LICENSE-2.0.txt)
 
-- org.apache.kafka:kafka-clients:2.0.1
+- org.apache.kafka:kafka-clients:2.2.0
diff --git a/flink-end-to-end-tests/flink-sql-client-test/pom.xml b/flink-end-to-end-tests/flink-sql-client-test/pom.xml
@@ -112,7 +112,7 @@ under the License.
  as we neither access nor package the kafka dependencies -->
  <groupId>org.apache.kafka</groupId>
  <artifactId>kafka-clients</artifactId>
- <version>2.0.1</version>
+ <version>2.2.0</version>
  </dependency>
  </dependencies>
  </dependencyManagement>

diff --git a/flink-end-to-end-tests/test-scripts/kafka_sql_common.sh b/flink-end-to-end-tests/test-scripts/kafka_sql_common.sh
@@ -17,13 +17,12 @@
 # limitations under the License.
 ################################################################################
 
-KAFKA_CONNECTOR_VERSION="$1"
-KAFKA_VERSION="$2"
-CONFLUENT_VERSION="$3"
-CONFLUENT_MAJOR_VERSION="$4"
-KAFKA_SQL_VERSION="$5"
+KAFKA_VERSION="$1"
+CONFLUENT_VERSION="$2"
+CONFLUENT_MAJOR_VERSION="$3"
+KAFKA_SQL_VERSION="$4"
 
-source "$(dirname "$0")"/kafka-common.sh $2 $3 $4
+source "$(dirname "$0")"/kafka-common.sh $1 $2 $3
 
 function create_kafka_json_source {
  topicName="$1"

diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client.sh b/flink-end-to-end-tests/test-scripts/test_sql_client.sh
@@ -19,15 +19,13 @@
 
 set -Eeuo pipefail
 
-KAFKA_CONNECTOR_VERSION="2.0"
-KAFKA_VERSION="2.0.1"
+KAFKA_VERSION="2.2.0"
 CONFLUENT_VERSION="5.0.0"
 CONFLUENT_MAJOR_VERSION="5.0"
 KAFKA_SQL_VERSION="universal"
 
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka_sql_common.sh \
- $KAFKA_CONNECTOR_VERSION \
  $KAFKA_VERSION \
  $CONFLUENT_VERSION \
  $CONFLUENT_MAJOR_VERSION \

diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka.sh
@@ -19,4 +19,4 @@
 
 set -Eeuo pipefail
 
-source "$(dirname "$0")"/test_sql_client_kafka_common.sh 2.0 2.0.1 5.0.0 5.0 "kafka" "universal"
+source "$(dirname "$0")"/test_sql_client_kafka_common.sh 2.2.0 5.0.0 5.0 "kafka" "universal"
diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka010.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka010.sh
@@ -19,4 +19,4 @@
 
 set -Eeuo pipefail
 
-source "$(dirname "$0")"/test_sql_client_kafka_common.sh 0.10 0.10.2.0 3.2.0 3.2 "kafka-0.10" "0.10"
+source "$(dirname "$0")"/test_sql_client_kafka_common.sh 0.10.2.0 3.2.0 3.2 "kafka-0.10" "0.10"
diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka011.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka011.sh
@@ -19,4 +19,4 @@
 
 set -Eeuo pipefail
 
-source "$(dirname "$0")"/test_sql_client_kafka_common.sh 0.11 0.11.0.2 3.2.0 3.2 "kafka-0.11" "0.11"
+source "$(dirname "$0")"/test_sql_client_kafka_common.sh 0.11.0.2 3.2.0 3.2 "kafka-0.11" "0.11"
diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client_kafka_common.sh b/flink-end-to-end-tests/test-scripts/test_sql_client_kafka_common.sh
@@ -17,16 +17,14 @@
 # limitations under the License.
 ################################################################################
 
-KAFKA_CONNECTOR_VERSION="$1"
-KAFKA_VERSION="$2"
-CONFLUENT_VERSION="$3"
-CONFLUENT_MAJOR_VERSION="$4"
-KAFKA_SQL_JAR="$5"
-KAFKA_SQL_VERSION="$6"
+KAFKA_VERSION="$1"
+CONFLUENT_VERSION="$2"
+CONFLUENT_MAJOR_VERSION="$3"
+KAFKA_SQL_JAR="$4"
+KAFKA_SQL_VERSION="$5"
 
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka_sql_common.sh \
- $KAFKA_CONNECTOR_VERSION \
  $KAFKA_VERSION \
  $CONFLUENT_VERSION \
  $CONFLUENT_MAJOR_VERSION \

diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh
@@ -20,6 +20,6 @@
 set -Eeuo pipefail
 
 source "$(dirname "$0")"/common.sh
-source "$(dirname "$0")"/kafka-common.sh 2.0.1 5.0.0 5.0
+source "$(dirname "$0")"/kafka-common.sh 2.2.0 5.0.0 5.0
 
 source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka-test/target/KafkaExample.jar
diff --git a/flink-jepsen/src/jepsen/flink/flink.clj b/flink-jepsen/src/jepsen/flink/flink.clj
@@ -35,7 +35,7 @@
 
 (def default-flink-dist-url "https://archive.apache.org/dist/flink/flink-1.6.0/flink-1.6.0-bin-hadoop28-scala_2.11.tgz")
 (def hadoop-dist-url "https://archive.apache.org/dist/hadoop/common/hadoop-2.8.3/hadoop-2.8.3.tar.gz")
-(def kafka-dist-url "http:https://mirror.funkfreundelandshut.de/apache/kafka/2.0.1/kafka_2.11-2.0.1.tgz")
+(def kafka-dist-url "http:https://mirror.funkfreundelandshut.de/apache/kafka/2.2.0/kafka_2.11-2.2.0.tgz")
 (def deb-zookeeper-package "3.4.9-3+deb9u1")
 (def deb-mesos-package "1.5.0-2.0.1")
 (def marathon-dist-url "https://downloads.mesosphere.io/marathon/builds/1.7.189-48bfd6000/marathon-1.7.189-48bfd6000.tgz")