[FLINK-25927][connectors][build] Consistent flink-connector-base usage

flink-connector-base was previously inconsistently used in connectors (directly shaded in some and transitively pulled in via flink-connector-files which was itself shaded in the table uber jar). FLINK-24687 moved flink-connector-files out from the flink-table uber jar. This commit implements a combined approach for ensuring a smooth transition for both Flink users and for external connector developers, as outlined in this: - all internal Flink connectors that depend on flink-connector-base now shade and relocate it - for compatibility, until external developers implement the same change, flink-connector-base is also included into flink-dist
mbalassi · Mar 14, 2022 · 9c77f13 · 9c77f13
1 parent 5464618
commit 9c77f13
Show file tree

Hide file tree

Showing 22 changed files with 342 additions and 17 deletions.
diff --git a/flink-connectors/flink-connector-aws-kinesis-firehose/pom.xml b/flink-connectors/flink-connector-aws-kinesis-firehose/pom.xml
@@ -46,12 +46,6 @@ under the License.
  <scope>provided</scope>
  </dependency>
 
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- </dependency>
-
  <dependency>
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-connector-aws-base</artifactId>
@@ -164,6 +158,34 @@ under the License.
  </execution>
  </executions>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-aws-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.firehose.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 </project>
diff --git a/flink-connectors/flink-connector-aws-kinesis-streams/pom.xml b/flink-connectors/flink-connector-aws-kinesis-streams/pom.xml
@@ -46,12 +46,6 @@ under the License.
  <scope>provided</scope>
  </dependency>
 
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- </dependency>
-
  <dependency>
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-connector-aws-base</artifactId>
@@ -145,6 +139,34 @@ under the License.
  </execution>
  </executions>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-aws-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.kinesis.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 </project>
diff --git a/flink-connectors/flink-connector-elasticsearch6/pom.xml b/flink-connectors/flink-connector-elasticsearch6/pom.xml
@@ -199,6 +199,33 @@ under the License.
  <forkCount>1</forkCount>
  </configuration>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-elasticsearch-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.elasticsearch6.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 </project>
diff --git a/flink-connectors/flink-connector-elasticsearch7/pom.xml b/flink-connectors/flink-connector-elasticsearch7/pom.xml
@@ -196,6 +196,33 @@ under the License.
  <forkCount>1</forkCount>
  </configuration>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ <include>org.apache.flink:flink-connector-elasticsearch-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.elasticsearch7.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 </project>
diff --git a/flink-connectors/flink-connector-files/pom.xml b/flink-connectors/flink-connector-files/pom.xml
@@ -152,6 +152,12 @@ under the License.
  <include>org.apache.flink:flink-connector-base</include>
  </includes>
  </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.files.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
  </configuration>
  </execution>
  </executions>

diff --git a/flink-connectors/flink-connector-hive/pom.xml b/flink-connectors/flink-connector-hive/pom.xml
@@ -549,6 +549,13 @@ under the License.
  <scope>test</scope>
  </dependency>
 
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
  <!--flink-java and flink-clients test dependencies used for HiveInputFormatTest-->
  <dependency>
  <groupId>org.apache.flink</groupId>

diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml
@@ -288,6 +288,33 @@ under the License.
  <argLine>-Xms256m -Xmx2048m -Dmvn.forkNumber=${surefire.forkNumber} -XX:-UseGCOverheadLimit -Duser.country=US -Duser.language=en</argLine>
  </configuration>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ </includes>
+ </artifactSet>
+ <shadeTestJar>true</shadeTestJar>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.kafka.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 

diff --git a/flink-connectors/flink-connector-pulsar/pom.xml b/flink-connectors/flink-connector-pulsar/pom.xml
@@ -340,6 +340,32 @@ under the License.
  </execution>
  </executions>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-base</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.pulsar.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 </project>
diff --git a/flink-connectors/flink-sql-connector-aws-kinesis-firehose/pom.xml b/flink-connectors/flink-sql-connector-aws-kinesis-firehose/pom.xml
@@ -54,7 +54,6 @@
  <configuration>
  <artifactSet>
  <includes>
- <include>org.apache.flink:flink-connector-base</include>
  <include>org.apache.flink:flink-connector-aws-base</include>
  <include>org.apache.flink:flink-connector-aws-kinesis-firehose</include>
  <include>software.amazon.awssdk:*</include>

diff --git a/flink-connectors/flink-sql-connector-aws-kinesis-streams/pom.xml b/flink-connectors/flink-sql-connector-aws-kinesis-streams/pom.xml
@@ -55,7 +55,6 @@
  <configuration>
  <artifactSet>
  <includes>
- <include>org.apache.flink:flink-connector-base</include>
  <include>org.apache.flink:flink-connector-aws-base</include>
  <include>org.apache.flink:flink-connector-aws-kinesis-streams</include>
  <include>software.amazon.awssdk:*</include>

diff --git a/flink-connectors/flink-sql-connector-hbase-1.4/pom.xml b/flink-connectors/flink-sql-connector-hbase-1.4/pom.xml
@@ -113,6 +113,10 @@ under the License.
  </filter>
  </filters>
  <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.sql.hbase14.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
  <!-- Force relocation of all HBase dependencies. -->
  <relocation>
  <pattern>org.apache.zookeeper</pattern>

diff --git a/flink-connectors/flink-sql-connector-hbase-2.2/pom.xml b/flink-connectors/flink-sql-connector-hbase-2.2/pom.xml
@@ -117,6 +117,10 @@ under the License.
  </filter>
  </filters>
  <relocations>
+ <relocation>
+ <pattern>org.apache.flink.connector.base</pattern>
+ <shadedPattern>org.apache.flink.connector.sql.hbase22.shaded.org.apache.flink.connector.base</shadedPattern>
+ </relocation>
  <!-- Force relocation of all HBase dependencies. -->
  <relocation>
  <pattern>org.apache.zookeeper</pattern>

diff --git a/flink-connectors/flink-sql-connector-kafka/pom.xml b/flink-connectors/flink-sql-connector-kafka/pom.xml
@@ -58,7 +58,6 @@ under the License.
  <configuration>
  <artifactSet>
  <includes>
- <include>org.apache.flink:flink-connector-base</include>
  <include>org.apache.flink:flink-connector-kafka</include>
  <include>org.apache.kafka:*</include>
  </includes>

diff --git a/flink-connectors/flink-sql-connector-kinesis/pom.xml b/flink-connectors/flink-sql-connector-kinesis/pom.xml
@@ -58,7 +58,6 @@ under the License.
  <configuration>
  <artifactSet>
  <includes>
- <include>org.apache.flink:flink-connector-base</include>
  <include>org.apache.flink:flink-connector-kinesis</include>
  <include>com.fasterxml.jackson.core:jackson-core</include>
  <include>com.fasterxml.jackson.core:jackson-databind</include>

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
@@ -138,6 +138,12 @@ under the License.
  </exclusions>
  </dependency>
 
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
  <!-- Default file system support. The Hadoop dependency -->
  <!-- is optional, so not being added to the dist jar -->
 

diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-firehose/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-firehose/pom.xml
@@ -51,6 +51,14 @@
  <scope>test</scope>
  <type>test-jar</type>
  </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-aws-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
  <dependency>
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-table-planner_${scala.binary.version}</artifactId>

diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-streams/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-aws-kinesis-streams/pom.xml
@@ -65,6 +65,13 @@
  <artifactId>jackson-databind</artifactId>
  <scope>test</scope>
  </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-aws-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
  </dependencies>
 
  <build>

diff --git a/flink-end-to-end-tests/flink-streaming-kinesis-test/pom.xml b/flink-end-to-end-tests/flink-streaming-kinesis-test/pom.xml
@@ -88,6 +88,13 @@ under the License.
  <type>test-jar</type>
  </dependency>
 
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-aws-base</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
  <dependency>
  <groupId>org.junit.jupiter</groupId>
  <artifactId>junit-jupiter</artifactId>