[FLINK-7840] [build] Shade netty in akka

spotify · Oct 26, 2017 · 8595dad · 8595dad
1 parent dc1a0dc
commit 8595dad
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 22 deletions.
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
@@ -427,17 +427,42 @@ under the License.
  <goal>shade</goal>
  </goals>
  <configuration>
+ <!-- we need this to avoid having to specify all of akka's dependencies -->
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
  <artifactSet>
  <includes combine.children="append">
  <include>org.apache.flink:flink-shaded-curator-recipes</include>
+ <!-- add akka, akka's netty, akka uncommon math -->
+ <!-- we can do this only because our own netty dependency is
+ already externally shaded (flink-shaded-netty) -->
+ <include>com.typesafe.akka:akka-remote_*</include>
+ <include>io.netty:netty</include>
+ <include>org.uncommons.maths:uncommons-maths</include>
  </includes>
  </artifactSet>
  <relocations combine.children="append">
+ <relocation>
+ <pattern>org.jboss.netty</pattern>
+ <shadedPattern>org.apache.flink.shaded.akka.org.jboss.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.uncommons.math</pattern>
+ <shadedPattern>org.apache.flink.shaded.akka.org.uncommons.math</shadedPattern>
+ </relocation>
  <relocation>
  <pattern>org.apache.curator</pattern>
  <shadedPattern>org.apache.flink.shaded.org.apache.curator</shadedPattern>
  </relocation>
  </relocations>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>META-INF/maven/io.netty/**</exclude>
+ <exclude>META-INF/maven/org.uncommons.maths/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
  </configuration>
  </execution>
  </executions>

diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml
@@ -64,6 +64,13 @@ under the License.
  <scope>compile</scope>
  </dependency>
 
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.10.6.Final</version>
+ <scope>compile</scope>
+ </dependency>
+
  <dependency>
  <groupId>junit</groupId>
  <artifactId>junit</artifactId>
@@ -117,6 +124,41 @@ under the License.
  <inherited>true</inherited>
  <extensions>true</extensions>
  </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes combine.children="append">
+ <include>io.netty:netty</include>
+ </includes>
+ </artifactSet>
+ <relocations combine.children="append">
+ <relocation>
+ <pattern>org.jboss.netty</pattern>
+ <shadedPattern>org.apache.flink.shaded.testutils.org.jboss.netty</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>META-INF/maven/io.netty/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 

diff --git a/...ent/flink-test-utils/src/main/java/org/apache/flink/networking/NetworkFailureHandler.java b/...ent/flink-test-utils/src/main/java/org/apache/flink/networking/NetworkFailureHandler.java
@@ -30,6 +30,7 @@
 import org.jboss.netty.channel.MessageEvent;
 import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
 import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml
@@ -26,7 +26,7 @@ under the License.
  <version>1.4-SNAPSHOT</version>
  <relativePath>..</relativePath>
  </parent>
- 
+
  <artifactId>flink-yarn_${scala.binary.version}</artifactId>
  <name>flink-yarn</name>
  <packaging>jar</packaging>
@@ -39,12 +39,6 @@ under the License.
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-runtime_${scala.binary.version}</artifactId>
  <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>hadoop-core</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- </exclusions>
  </dependency>
 
  <dependency>
@@ -59,22 +53,16 @@ under the License.
  <version>${project.version}</version>
  </dependency>
 
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
  <dependency>
  <groupId>com.typesafe.akka</groupId>
  <artifactId>akka-actor_${scala.binary.version}</artifactId>
+ <scope>provided</scope>
  </dependency>
 
  <dependency>
  <groupId>com.typesafe.akka</groupId>
  <artifactId>akka-remote_${scala.binary.version}</artifactId>
+ <scope>provided</scope>
  </dependency>
 
  <dependency>
@@ -92,14 +80,16 @@ under the License.
  </exclusions>
  </dependency>
 
+ <!-- test dependencies -->
+
  <dependency>
- <groupId>com.typesafe.akka</groupId>
- <artifactId>akka-testkit_${scala.binary.version}</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
  <scope>test</scope>
  </dependency>
 
- <!-- test dependencies -->
-
  <dependency>
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-runtime_${scala.binary.version}</artifactId>
@@ -123,6 +113,12 @@ under the License.
  <type>test-jar</type>
  <version>${hadoop.version}</version>
  </dependency>
+
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-testkit_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
  </dependencies>
 
  <profiles>

diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
@@ -308,10 +308,18 @@ check_shaded_artifacts() {
  return 1
  fi
 
- NETTY=`cat allClasses | grep '^io/netty' | wc -l`
- if [ "$NETTY" != "0" ]; then
+ IO_NETTY=`cat allClasses | grep '^io/netty' | wc -l`
+ if [ "$IO_NETTY" != "0" ]; then
  echo "=============================================================================="
- echo "Detected '$NETTY' unshaded netty dependencies in fat jar"
+ echo "Detected '$IO_NETTY' unshaded io.netty classes in fat jar"
+ echo "=============================================================================="
+ return 1
+ fi
+
+ ORG_NETTY=`cat allClasses | grep '^org/jboss/netty' | wc -l`
+ if [ "$ORG_NETTY" != "0" ]; then
+ echo "=============================================================================="
+ echo "Detected '$ORG_NETTY' unshaded org.jboss.netty classes in fat jar"
  echo "=============================================================================="
  return 1
  fi