[FLINK-10922] Refactor the placement of the Flink Kafka connector end…

… to end test module
bbejeck · Nov 19, 2018 · b33e2c0 · b33e2c0
1 parent b7bbd06
commit b33e2c0
Show file tree

Hide file tree

Showing 19 changed files with 49 additions and 91 deletions.
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
@@ -187,48 +187,6 @@ under the License.
  </excludes>
  </fileSet>
 
- <!-- copy jar files of the streaming kafka examples -->
- <fileSet>
- <directory>../flink-examples/flink-examples-streaming-kafka/target</directory>
- <outputDirectory>examples/streaming</outputDirectory>
- <fileMode>0644</fileMode>
- <includes>
- <include>*.jar</include>
- </includes>
- <excludes>
- <exclude>flink-examples-streaming-kafka*.jar</exclude>
- <exclude>original-*.jar</exclude>
- </excludes>
- </fileSet>
-
- <!-- copy jar files of the streaming kafka 0.10 examples -->
- <fileSet>
- <directory>../flink-examples/flink-examples-streaming-kafka-0.10/target</directory>
- <outputDirectory>examples/streaming</outputDirectory>
- <fileMode>0644</fileMode>
- <includes>
- <include>*.jar</include>
- </includes>
- <excludes>
- <exclude>flink-examples-streaming-kafka*.jar</exclude>
- <exclude>original-*.jar</exclude>
- </excludes>
- </fileSet>
-
- <!-- copy jar files of the streaming kafka 0.11 examples -->
- <fileSet>
- <directory>../flink-examples/flink-examples-streaming-kafka-0.11/target</directory>
- <outputDirectory>examples/streaming</outputDirectory>
- <fileMode>0644</fileMode>
- <includes>
- <include>*.jar</include>
- </includes>
- <excludes>
- <exclude>flink-examples-streaming-kafka*.jar</exclude>
- <exclude>original-*.jar</exclude>
- </excludes>
- </fileSet>
-
  <!-- copy jar files of the gelly examples -->
  <fileSet>
  <directory>../flink-libraries/flink-gelly-examples/target</directory>

diff --git a/...ink-examples-streaming-kafka-base/pom.xml → ...s/flink-streaming-kafka-test-base/pom.xml b/...ink-examples-streaming-kafka-base/pom.xml → ...s/flink-streaming-kafka-test-base/pom.xml
@@ -21,14 +21,14 @@ under the License.
  xmlns:xsi="http:https://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http:https://maven.apache.org/POM/4.0.0 http:https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <parent>
- <artifactId>flink-examples</artifactId>
+ <artifactId>flink-end-to-end-tests</artifactId>
  <groupId>org.apache.flink</groupId>
  <version>1.8-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 
- <artifactId>flink-examples-streaming-kafka-base_${scala.binary.version}</artifactId>
- <name>flink-examples-streaming-kafka-base</name>
+ <artifactId>flink-streaming-kafka-test-base_${scala.binary.version}</artifactId>
+ <name>flink-streaming-kafka-test-base</name>
 
  <packaging>jar</packaging>
 

diff --git a/.../kafka/base/CustomWatermarkExtractor.java → ...a/test/base/CustomWatermarkExtractor.java b/.../kafka/base/CustomWatermarkExtractor.java → ...a/test/base/CustomWatermarkExtractor.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka.base;
+package org.apache.flink.streaming.kafka.test.base;
 
 import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
 import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor;

diff --git a/...aming/examples/kafka/base/KafkaEvent.java → ...streaming/kafka/test/base/KafkaEvent.java b/...aming/examples/kafka/base/KafkaEvent.java → ...streaming/kafka/test/base/KafkaEvent.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka.base;
+package org.apache.flink.streaming.kafka.test.base;
 
 /**
  * The event type used in the {@link Kafka010Example}.

diff --git a/...examples/kafka/base/KafkaEventSchema.java → ...ing/kafka/test/base/KafkaEventSchema.java b/...examples/kafka/base/KafkaEventSchema.java → ...ing/kafka/test/base/KafkaEventSchema.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka.base;
+package org.apache.flink.streaming.kafka.test.base;
 
 import org.apache.flink.api.common.serialization.DeserializationSchema;
 import org.apache.flink.api.common.serialization.SerializationSchema;

diff --git a/...examples/kafka/base/KafkaExampleUtil.java → ...ing/kafka/test/base/KafkaExampleUtil.java b/...examples/kafka/base/KafkaExampleUtil.java → ...ing/kafka/test/base/KafkaExampleUtil.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka.base;
+package org.apache.flink.streaming.kafka.test.base;
 
 import org.apache.flink.api.common.restartstrategy.RestartStrategies;
 import org.apache.flink.api.java.utils.ParameterTool;

diff --git a/...les/kafka/base/RollingAdditionMapper.java → ...afka/test/base/RollingAdditionMapper.java b/...les/kafka/base/RollingAdditionMapper.java → ...afka/test/base/RollingAdditionMapper.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka.base;
+package org.apache.flink.streaming.kafka.test.base;
 
 import org.apache.flink.api.common.functions.RichMapFunction;
 import org.apache.flink.api.common.state.ValueState;

diff --git a/...es/flink-examples-streaming-kafka/pom.xml → ...-tests/flink-streaming-kafka-test/pom.xml b/...es/flink-examples-streaming-kafka/pom.xml → ...-tests/flink-streaming-kafka-test/pom.xml
@@ -21,19 +21,19 @@ under the License.
  xmlns:xsi="http:https://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http:https://maven.apache.org/POM/4.0.0 http:https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <parent>
- <artifactId>flink-examples</artifactId>
+ <artifactId>flink-end-to-end-tests</artifactId>
  <groupId>org.apache.flink</groupId>
  <version>1.8-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 
- <artifactId>flink-examples-streaming-kafka_${scala.binary.version}</artifactId>
- <name>flink-examples-streaming-kafka</name>
+ <artifactId>flink-streaming-kafka-test_${scala.binary.version}</artifactId>
+ <name>flink-streaming-kafka-test</name>
 
  <dependencies>
  <dependency>
  <groupId>org.apache.flink</groupId>
- <artifactId>flink-examples-streaming-kafka-base_${scala.binary.version}</artifactId>
+ <artifactId>flink-streaming-kafka-test-base_${scala.binary.version}</artifactId>
  <version>${project.version}</version>
  </dependency>
 
@@ -75,7 +75,7 @@ under the License.
  <createDependencyReducedPom>false</createDependencyReducedPom>
  <transformers>
  <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.flink.streaming.examples.kafka.KafkaExample</mainClass>
+ <mainClass>org.apache.flink.streaming.kafka.test.KafkaExample</mainClass>
  </transformer>
  </transformers>
  <finalName>KafkaExample</finalName>

diff --git a/...treaming/examples/kafka/KafkaExample.java → ...nk/streaming/kafka/test/KafkaExample.java b/...treaming/examples/kafka/KafkaExample.java → ...nk/streaming/kafka/test/KafkaExample.java
@@ -15,18 +15,18 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka;
+package org.apache.flink.streaming.kafka.test;
 
 import org.apache.flink.api.java.utils.ParameterTool;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
-import org.apache.flink.streaming.examples.kafka.base.CustomWatermarkExtractor;
-import org.apache.flink.streaming.examples.kafka.base.KafkaEvent;
-import org.apache.flink.streaming.examples.kafka.base.KafkaEventSchema;
-import org.apache.flink.streaming.examples.kafka.base.KafkaExampleUtil;
-import org.apache.flink.streaming.examples.kafka.base.RollingAdditionMapper;
+import org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor;
+import org.apache.flink.streaming.kafka.test.base.KafkaEvent;
+import org.apache.flink.streaming.kafka.test.base.KafkaEventSchema;
+import org.apache.flink.streaming.kafka.test.base.KafkaExampleUtil;
+import org.apache.flink.streaming.kafka.test.base.RollingAdditionMapper;
 import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper;
 
 /**

diff --git a/...ink-examples-streaming-kafka-0.10/pom.xml → ...sts/flink-streaming-kafka010-test/pom.xml b/...ink-examples-streaming-kafka-0.10/pom.xml → ...sts/flink-streaming-kafka010-test/pom.xml
@@ -21,19 +21,19 @@ under the License.
  xmlns:xsi="http:https://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http:https://maven.apache.org/POM/4.0.0 http:https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <parent>
- <artifactId>flink-examples</artifactId>
+ <artifactId>flink-end-to-end-tests</artifactId>
  <groupId>org.apache.flink</groupId>
  <version>1.8-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 
- <artifactId>flink-examples-streaming-kafka-0.10_${scala.binary.version}</artifactId>
- <name>flink-examples-streaming-kafka-0.10</name>
+ <artifactId>flink-streaming-kafka010-test_${scala.binary.version}</artifactId>
+ <name>flink-streaming-kafka010-test</name>
 
  <dependencies>
  <dependency>
  <groupId>org.apache.flink</groupId>
- <artifactId>flink-examples-streaming-kafka-base_${scala.binary.version}</artifactId>
+ <artifactId>flink-streaming-kafka-test-base_${scala.binary.version}</artifactId>
  <version>${project.version}</version>
  </dependency>
 
@@ -119,7 +119,7 @@ under the License.
  <createDependencyReducedPom>false</createDependencyReducedPom>
  <transformers>
  <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.flink.streaming.examples.kafka.Kafka010Example</mainClass>
+ <mainClass>org.apache.flink.streaming.kafka.test.Kafka010Example</mainClass>
  </transformer>
  </transformers>
  <finalName>Kafka010Example</finalName>

diff --git a/...aming/examples/kafka/Kafka010Example.java → ...streaming/kafka/test/Kafka010Example.java b/...aming/examples/kafka/Kafka010Example.java → ...streaming/kafka/test/Kafka010Example.java
@@ -15,18 +15,18 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka;
+package org.apache.flink.streaming.kafka.test;
 
 import org.apache.flink.api.java.utils.ParameterTool;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010;
-import org.apache.flink.streaming.examples.kafka.base.CustomWatermarkExtractor;
-import org.apache.flink.streaming.examples.kafka.base.KafkaEvent;
-import org.apache.flink.streaming.examples.kafka.base.KafkaEventSchema;
-import org.apache.flink.streaming.examples.kafka.base.KafkaExampleUtil;
-import org.apache.flink.streaming.examples.kafka.base.RollingAdditionMapper;
+import org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor;
+import org.apache.flink.streaming.kafka.test.base.KafkaEvent;
+import org.apache.flink.streaming.kafka.test.base.KafkaEventSchema;
+import org.apache.flink.streaming.kafka.test.base.KafkaExampleUtil;
+import org.apache.flink.streaming.kafka.test.base.RollingAdditionMapper;
 
 /**
  * A simple example that shows how to read from and write to Kafka. This will read String messages

diff --git a/...cala/examples/kafka/Kafka010Example.scala → ...ng/scala/kafka/test/Kafka010Example.scala b/...cala/examples/kafka/Kafka010Example.scala → ...ng/scala/kafka/test/Kafka010Example.scala
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.scala.examples.kafka
+package org.apache.flink.streaming.scala.kafka.test
 
 import org.apache.flink.api.common.restartstrategy.RestartStrategies
 import org.apache.flink.api.common.serialization.SimpleStringSchema

diff --git a/...ink-examples-streaming-kafka-0.11/pom.xml → ...sts/flink-streaming-kafka011-test/pom.xml b/...ink-examples-streaming-kafka-0.11/pom.xml → ...sts/flink-streaming-kafka011-test/pom.xml
@@ -21,20 +21,20 @@ under the License.
  xmlns:xsi="http:https://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http:https://maven.apache.org/POM/4.0.0 http:https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <parent>
- <artifactId>flink-examples</artifactId>
+ <artifactId>flink-end-to-end-tests</artifactId>
  <groupId>org.apache.flink</groupId>
  <version>1.8-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 
- <artifactId>flink-examples-streaming-kafka-0.11_${scala.binary.version}</artifactId>
- <name>flink-examples-streaming-kafka-0.11</name>
+ <artifactId>flink-streaming-kafka011-test_${scala.binary.version}</artifactId>
+ <name>flink-streaming-kafka011-test</name>
  <packaging>jar</packaging>
 
  <dependencies>
  <dependency>
  <groupId>org.apache.flink</groupId>
- <artifactId>flink-examples-streaming-kafka-base_${scala.binary.version}</artifactId>
+ <artifactId>flink-streaming-kafka-test-base_${scala.binary.version}</artifactId>
  <version>${project.version}</version>
  </dependency>
 
@@ -76,7 +76,7 @@ under the License.
  <createDependencyReducedPom>false</createDependencyReducedPom>
  <transformers>
  <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.flink.streaming.examples.kafka.Kafka011Example</mainClass>
+ <mainClass>org.apache.flink.streaming.kafka.test.Kafka011Example</mainClass>
  </transformer>
  </transformers>
  <finalName>Kafka011Example</finalName>

diff --git a/...aming/examples/kafka/Kafka011Example.java → ...streaming/kafka/test/Kafka011Example.java b/...aming/examples/kafka/Kafka011Example.java → ...streaming/kafka/test/Kafka011Example.java
@@ -15,18 +15,18 @@
  * limitations under the License.
  */
 
-package org.apache.flink.streaming.examples.kafka;
+package org.apache.flink.streaming.kafka.test;
 
 import org.apache.flink.api.java.utils.ParameterTool;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011;
-import org.apache.flink.streaming.examples.kafka.base.CustomWatermarkExtractor;
-import org.apache.flink.streaming.examples.kafka.base.KafkaEvent;
-import org.apache.flink.streaming.examples.kafka.base.KafkaEventSchema;
-import org.apache.flink.streaming.examples.kafka.base.KafkaExampleUtil;
-import org.apache.flink.streaming.examples.kafka.base.RollingAdditionMapper;
+import org.apache.flink.streaming.kafka.test.base.CustomWatermarkExtractor;
+import org.apache.flink.streaming.kafka.test.base.KafkaEvent;
+import org.apache.flink.streaming.kafka.test.base.KafkaEventSchema;
+import org.apache.flink.streaming.kafka.test.base.KafkaExampleUtil;
+import org.apache.flink.streaming.kafka.test.base.RollingAdditionMapper;
 
 /**
  * A simple example that shows how to read from and write to Kafka. This will read String messages

diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
@@ -61,6 +61,10 @@ under the License.
  <module>flink-metrics-availability-test</module>
  <module>flink-metrics-reporter-prometheus-test</module>
  <module>flink-heavy-deployment-stress-test</module>
+ <module>flink-streaming-kafka-test-base</module>
+ <module>flink-streaming-kafka-test</module>
+ <module>flink-streaming-kafka011-test</module>
+ <module>flink-streaming-kafka010-test</module>
  </modules>
 
  <build>

diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka.sh
@@ -22,4 +22,4 @@ set -Eeuo pipefail
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka-common.sh 2.0.1 5.0.0 5.0
 
-source "$(dirname "$0")"/test_streaming_kafka_common.sh $FLINK_DIR/examples/streaming/KafkaExample.jar
+source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka-test/target/KafkaExample.jar
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh
@@ -22,5 +22,5 @@ set -Eeuo pipefail
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka-common.sh 0.10.2.0 3.2.0 3.2
 
-source "$(dirname "$0")"/test_streaming_kafka_common.sh $FLINK_DIR/examples/streaming/Kafka010Example.jar
+source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka010-test/target/Kafka010Example.jar
 
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kafka011.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka011.sh
@@ -20,5 +20,5 @@
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/kafka-common.sh 0.11.0.2 3.2.0 3.2
 
-source "$(dirname "$0")"/test_streaming_kafka_common.sh $FLINK_DIR/examples/streaming/Kafka011Example.jar
+source "$(dirname "$0")"/test_streaming_kafka_common.sh ${END_TO_END_DIR}/flink-streaming-kafka011-test/target/Kafka011Example.jar
 
diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml
@@ -35,11 +35,7 @@ under the License.
  <modules>
  <module>flink-examples-batch</module>
  <module>flink-examples-streaming</module>
- <module>flink-examples-streaming-kafka-base</module>
- <module>flink-examples-streaming-kafka</module>
- <module>flink-examples-streaming-kafka-0.10</module>
  <module>flink-examples-table</module>
- <module>flink-examples-streaming-kafka-0.11</module>
  </modules>
 
  <dependencies>