From 84e76f4d3274e07176f7377b7b739b6f180c6296 Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Tue, 24 Feb 2015 16:30:02 +0100 Subject: [PATCH] [FLINK-1605] Bundle all hadoop dependencies and shade guava away This closes #454 --- flink-clients/pom.xml | 4 - flink-compiler/pom.xml | 2 - flink-core/pom.xml | 41 +- flink-dist/pom.xml | 136 ++--- flink-dist/src/main/assemblies/bin.xml | 39 +- flink-dist/src/main/assemblies/yarn.xml | 158 ------ .../main/flink-bin/yarn-bin/yarn-session.sh | 2 +- flink-java/pom.xml | 69 +-- .../resources/archetype-resources/pom.xml | 5 - flink-quickstart/pom.xml | 4 +- flink-runtime/pom.xml | 52 +- .../apache/flink/runtime/akka/AkkaUtils.scala | 3 - .../src/test/resources/log4j-test.properties | 2 +- flink-scala/pom.xml | 3 +- .../flink-shaded-hadoop1/pom.xml | 83 +++ .../flink-shaded-hadoop2/pom.xml | 195 +++++++ .../flink-shaded-include-yarn-tests/pom.xml | 78 +++ .../flink-shaded-include-yarn/pom.xml | 306 ++++++++++ {flink-shaded => flink-shaded-hadoop}/pom.xml | 66 ++- flink-staging/flink-expressions/pom.xml | 1 - flink-staging/flink-gelly/pom.xml | 36 +- .../flink-hadoop-compatibility/pom.xml | 25 +- flink-staging/flink-hbase/pom.xml | 58 +- .../flink-streaming-connectors/pom.xml | 31 +- .../flink-streaming-examples/pom.xml | 24 +- .../flink-streaming-scala/pom.xml | 3 +- flink-staging/flink-tachyon/pom.xml | 16 +- flink-test-utils/pom.xml | 2 - flink-tests/pom.xml | 38 +- flink-yarn-tests/pom.xml | 96 +++- .../java/org/apache/flink/yarn/UtilsTest.java | 2 +- .../YARNSessionCapacitySchedulerITCase.java | 0 .../flink/yarn/YARNSessionFIFOITCase.java | 3 +- .../org/apache/flink/yarn/YarnTestBase.java | 10 +- .../resources/log4j-test.properties | 0 flink-yarn/pom.xml | 34 +- .../apache/flink/yarn/FlinkYarnClient.java | 10 + pom.xml | 531 +++--------------- tools/travis_mvn_watchdog.sh | 2 +- 39 files changed, 1030 insertions(+), 1140 deletions(-) delete mode 100644 flink-dist/src/main/assemblies/yarn.xml create mode 100644 flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml create mode 100644 flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml create mode 100644 flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml create mode 100644 flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml rename {flink-shaded => flink-shaded-hadoop}/pom.xml (55%) rename flink-yarn-tests/src/{test => main}/java/org/apache/flink/yarn/UtilsTest.java (96%) rename flink-yarn-tests/src/{test => main}/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java (100%) rename flink-yarn-tests/src/{test => main}/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java (98%) rename flink-yarn-tests/src/{test => main}/java/org/apache/flink/yarn/YarnTestBase.java (97%) rename flink-yarn-tests/src/{test => main}/resources/log4j-test.properties (100%) diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml index 1a96c9c5330dd..d7dccade84a0f 100644 --- a/flink-clients/pom.xml +++ b/flink-clients/pom.xml @@ -60,12 +60,10 @@ under the License. ${project.version} - com.google.guava guava ${guava.version} - provided @@ -99,8 +97,6 @@ under the License. commons-io commons-io - 2.4 - compile diff --git a/flink-compiler/pom.xml b/flink-compiler/pom.xml index 74dae55f9f41c..46d5a9e728bfb 100644 --- a/flink-compiler/pom.xml +++ b/flink-compiler/pom.xml @@ -53,12 +53,10 @@ under the License. ${project.version} - com.google.guava guava ${guava.version} - provided diff --git a/flink-core/pom.xml b/flink-core/pom.xml index 182a77abde741..294a4fe65dda0 100644 --- a/flink-core/pom.xml +++ b/flink-core/pom.xml @@ -37,7 +37,7 @@ under the License. org.apache.flink - flink-shaded + ${shading-artifact.name} ${project.version} @@ -55,15 +55,13 @@ under the License. com.twitter chill_2.10 - 0.5.1 + ${chill.version} - com.google.guava guava ${guava.version} - provided @@ -83,39 +81,4 @@ under the License. - - - - hadoop-1 - - - - hadoop.profile1 - - - - - - org.apache.hadoop - hadoop-core - - - - - hadoop-2 - - - - !hadoop.profile - - - - - org.apache.hadoop - hadoop-common - - - - - diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index af3b100e50e23..a39c46ec68142 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -35,18 +35,6 @@ under the License. - - org.apache.flink - flink-shaded - ${project.version} - - - com.google.guava - guava - - - - org.apache.flink flink-core @@ -122,22 +110,6 @@ under the License. - - hadoop-1 - - - - hadoop.profile1 - - - - ${hadoop-one.version} - - - - - - include-yarn @@ -153,80 +125,6 @@ under the License. ${project.version} - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - package - - shade - - - false - ${project.artifactId}-${project.version}-yarn-uberjar - - - org.apache.flink:flink-java-examples - org.apache.flink:flink-scala-examples - org.apache.flink:flink-streaming-examples - - - - - org.apache.flink:flink-clients - - - web-docs/** - - - - - - - reference.conf - - - - - - org.apache.flink.client.FlinkYarnSessionCli - - - - - - - - - maven-assembly-plugin - 2.4 - - - - yarn-bin - package - - single - - - - src/main/assemblies/yarn.xml - - flink-${project.version}-bin - false - - - - - - @@ -361,6 +259,40 @@ under the License. + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + false + false + ${project.artifactId}-${project.version} + + + org.apache.flink:flink-java-examples + org.apache.flink:flink-scala-examples + org.apache.flink:flink-streaming-examples + + + + + reference.conf + + + + + + + + + maven-assembly-plugin diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml index cd85d3f44d797..64fca1b9f86ec 100644 --- a/flink-dist/src/main/assemblies/bin.xml +++ b/flink-dist/src/main/assemblies/bin.xml @@ -28,29 +28,14 @@ under the License. true flink-${project.version} - - - - lib - false - true - false - false - true - - - org.apache.flink:flink-java-examples - org.apache.flink:flink-scala-examples - org.apache.flink:flink-streaming-examples - - - - - + + + target/flink-dist-${project.version}.jar + lib/ + 0644 + + src/main/resources/flink-conf.yaml conf @@ -65,15 +50,17 @@ under the License. bin 0755 - + + + src/main/flink-bin/yarn-bin + bin + 0755 + src/main/flink-bin/conf conf 0644 - - log4j-yarn-session.properties - diff --git a/flink-dist/src/main/assemblies/yarn.xml b/flink-dist/src/main/assemblies/yarn.xml deleted file mode 100644 index 99ebaead36892..0000000000000 --- a/flink-dist/src/main/assemblies/yarn.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - yarn - - dir - - - true - flink-yarn-${project.version} - - - - - src/main/flink-bin/yarn-bin - bin - 0755 - - - - src/main/flink-bin/ - log - 0644 - - **/* - - - - - - src/main/flink-bin/ - ship - 0644 - - **/* - - - - - - src/main/flink-bin/ - - 0644 - - *.txt - *.properties - LICENSE* - NOTICE* - - - - - - src/main/flink-bin/tools - tools - 0644 - - - ../flink-clients/src/main/resources/web-docs - tools - 0644 - - *.html - img/delete-icon.png - img/GradientBoxes.png - img/gradient.jpg - - - - - - ../flink-examples/flink-java-examples/target - examples - 0644 - - *.jar - - - flink-java-examples-${project.version}.jar - original-flink-java-examples-${project.version}.jar - flink-java-examples-${project.version}-sources.jar - - - - - - - - - src/main/resources/flink-conf.yaml - conf - 0644 - - - src/main/flink-bin/conf/logback-yarn.xml - conf - 0644 - - - src/main/flink-bin/conf/log4j.properties - conf - 0644 - - - src/main/flink-bin/conf/log4j-cli.properties - conf - 0644 - - - src/main/flink-bin/conf/log4j-yarn-session.properties - conf - 0644 - - - src/main/flink-bin/conf/logback.xml - conf - 0644 - - - - - target/flink-dist-${project.version}-yarn-uberjar.jar - lib/ - 0644 - - - - src/main/flink-bin/bin/flink - bin/ - 0755 - - - src/main/flink-bin/bin/config.sh - bin/ - 0755 - - - diff --git a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh index 3ff2972cd1827..9bf3d0ccae414 100755 --- a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh +++ b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh @@ -52,5 +52,5 @@ log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4 export FLINK_CONF_DIR -$JAVA_RUN $JVM_ARGS -classpath $CC_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR $log_setting org.apache.flink.client.FlinkYarnSessionCli -ship $bin/../ship/ -j $FLINK_LIB_DIR/*yarn-uberjar.jar $* +$JAVA_RUN $JVM_ARGS -classpath $CC_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR $log_setting org.apache.flink.client.FlinkYarnSessionCli -ship $bin/../lib/ -j $FLINK_LIB_DIR/flink-dist*.jar $* diff --git a/flink-java/pom.xml b/flink-java/pom.xml index 63d30a475a23e..34fcdd2675117 100644 --- a/flink-java/pom.xml +++ b/flink-java/pom.xml @@ -33,9 +33,6 @@ under the License. flink-java jar - - 0.5.2 - @@ -43,6 +40,12 @@ under the License. flink-core ${project.version} + + + org.apache.flink + ${shading-artifact.name} + ${project.version} + org.apache.avro @@ -58,6 +61,7 @@ under the License. org.ow2.asm asm + ${asm.version} @@ -117,12 +121,10 @@ under the License. joda-time - com.google.guava guava ${guava.version} - provided @@ -150,61 +152,4 @@ under the License. - - - - - hadoop-1 - - - - hadoop.profile1 - - - - - - org.apache.hadoop - hadoop-core - - - - - - - - - - - hadoop-2 - - - - !hadoop.profile - - - - - org.apache.hadoop - hadoop-common - - - - - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - - - - - - - - diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index adc6ab5cb93c1..d84e922382853 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -59,11 +59,6 @@ under the License. flink-streaming-core 0.9-SNAPSHOT - - org.apache.flink - flink-streaming-connectors - 0.9-SNAPSHOT - org.apache.flink flink-clients diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml index dd3a903a50487..78b99ad181cc1 100644 --- a/flink-quickstart/pom.xml +++ b/flink-quickstart/pom.xml @@ -62,8 +62,7 @@ under the License. ${skipTests} - - + org.apache.maven.plugins maven-shade-plugin @@ -73,7 +72,6 @@ under the License. - \ No newline at end of file diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index 59ec7f09f939c..14e27aa5ea4bf 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -46,11 +46,22 @@ under the License. flink-java ${project.version} + + + org.apache.flink + ${shading-artifact.name} + ${project.version} + commons-cli commons-cli + + + commons-io + commons-io + org.eclipse.jetty @@ -88,12 +99,10 @@ under the License. 1.1 - com.google.guava guava ${guava.version} - provided @@ -338,43 +347,4 @@ under the License. - - - - - hadoop-1 - - - - hadoop.profile1 - - - - - - org.apache.hadoop - hadoop-core - - - - - hadoop-2 - - - - !hadoop.profile - - - - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-hdfs - - - - diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala index 014cf5791d706..be55e147a9788 100644 --- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala +++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala @@ -232,9 +232,6 @@ object AkkaUtils { | } | | remote { - | quarantine-systems-for = off - | gate-invalid-addresses-for = 5 s - | | startup-timeout = $startupTimeout | | transport-failure-detector{ diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties index 491b219221c44..76b237e7e92a2 100644 --- a/flink-runtime/src/test/resources/log4j-test.properties +++ b/flink-runtime/src/test/resources/log4j-test.properties @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -log4j.rootLogger=OFF +log4j.rootLogger=OFF, console # ----------------------------------------------------------------------------- # Console (use 'console') diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml index 3b686abf5312a..32faa6fe743a5 100644 --- a/flink-scala/pom.xml +++ b/flink-scala/pom.xml @@ -75,14 +75,13 @@ under the License. org.ow2.asm asm + ${asm.version} - com.google.guava guava ${guava.version} - provided diff --git a/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml new file mode 100644 index 0000000000000..1264f74e581a5 --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml @@ -0,0 +1,83 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-shaded-hadoop + 0.9-SNAPSHOT + .. + + + flink-shaded-hadoop1 + flink-shaded-hadoop1 + + jar + + + + org.apache.hadoop + hadoop-core + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + + + + diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml new file mode 100644 index 0000000000000..7a441c66ae507 --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml @@ -0,0 +1,195 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-shaded-hadoop + 0.9-SNAPSHOT + .. + + + flink-shaded-hadoop2 + flink-shaded-hadoop2 + + jar + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + org.apache.hadoop + hadoop-yarn-common + + + com.google.inject.extensions + guice-servlet + + + io.netty + netty + + + + + + diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml new file mode 100644 index 0000000000000..e271c88eae7e7 --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml @@ -0,0 +1,78 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-shaded-hadoop + 0.9-SNAPSHOT + .. + + + flink-shaded-include-yarn-tests + flink-shaded-include-yarn-tests + + jar + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-server-tests + ${hadoop.version} + + + + + org.apache.hadoop + hadoop-yarn-server-tests + ${hadoop.version} + test-jar + + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + ${hadoop.version} + + + + + diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml new file mode 100644 index 0000000000000..c8e9aa4ea5247 --- /dev/null +++ b/flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml @@ -0,0 +1,306 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-shaded-hadoop + 0.9-SNAPSHOT + .. + + + flink-shaded-include-yarn + flink-shaded-include-yarn + + jar + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + org.apache.hadoop + hadoop-yarn-common + + + com.google.inject.extensions + guice-servlet + + + io.netty + netty + + + + + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + org.apache.hadoop + hadoop-yarn-common + + + com.google.inject.extensions + guice-servlet + + + io.netty + netty + + + + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + asm + asm + + + org.ow2.asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + org.apache.hadoop + hadoop-yarn-common + + + com.google.inject.extensions + guice-servlet + + + io.netty + netty + + + + + diff --git a/flink-shaded/pom.xml b/flink-shaded-hadoop/pom.xml similarity index 55% rename from flink-shaded/pom.xml rename to flink-shaded-hadoop/pom.xml index 965e6fa953b23..60d81abe2d3fa 100644 --- a/flink-shaded/pom.xml +++ b/flink-shaded-hadoop/pom.xml @@ -29,29 +29,42 @@ under the License. .. - flink-shaded - flink-shaded + flink-shaded-hadoop + flink-shaded-hadoop - jar + pom - - - com.google.guava - guava - ${guava.version} - compile - - + + ${shading-artifact.name} + + + + + include-yarn + + + + !hadoop.profile + + + + flink-shaded-include-yarn-tests + + + - + org.apache.maven.plugins maven-shade-plugin 2.3 + shade-hadoop package shade @@ -60,19 +73,34 @@ under the License. false true ${project.basedir}/target/dependency-reduced-pom.xml - - - com.google.guava:guava - - + + + org.slf4j:* + + org/slf4j/impl/StaticLoggerBinder* + + + + + + + com.google - org.apache.flink.shaded.com.google + org.apache.flink.hadoop.shaded.com.google - com.google.protobuf.** + com.google.inject.** + + org.objectweb.asm + org.apache.flink.hadoop.shaded.org.objectweb.asm + + + org.jboss.netty + org.apache.flink.hadoop.shaded.org.jboss.netty + diff --git a/flink-staging/flink-expressions/pom.xml b/flink-staging/flink-expressions/pom.xml index 800cc2570b3ed..56ed34596348b 100644 --- a/flink-staging/flink-expressions/pom.xml +++ b/flink-staging/flink-expressions/pom.xml @@ -38,7 +38,6 @@ under the License. com.google.guava guava ${guava.version} - provided diff --git a/flink-staging/flink-gelly/pom.xml b/flink-staging/flink-gelly/pom.xml index 13ed002161e4b..a36ab4bcc715f 100755 --- a/flink-staging/flink-gelly/pom.xml +++ b/flink-staging/flink-gelly/pom.xml @@ -51,36 +51,10 @@ under the License. ${project.version} test + + com.google.guava + guava + ${guava.version} + - - - - - hadoop-1 - - - - hadoop.profile1 - - - - - - com.google.guava - guava - ${guava.version} - provided - - - - - hadoop-2 - - - - !hadoop.profile - - - - diff --git a/flink-staging/flink-hadoop-compatibility/pom.xml b/flink-staging/flink-hadoop-compatibility/pom.xml index e39f8b678eee1..d50b98123286e 100644 --- a/flink-staging/flink-hadoop-compatibility/pom.xml +++ b/flink-staging/flink-hadoop-compatibility/pom.xml @@ -46,6 +46,13 @@ under the License. flink-clients ${project.version} + + + org.apache.flink + ${shading-artifact.name} + ${project.version} + + org.apache.flink flink-tests @@ -59,22 +66,4 @@ under the License. test - - - - hadoop-2 - - - - !hadoop.profile - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - - diff --git a/flink-staging/flink-hbase/pom.xml b/flink-staging/flink-hbase/pom.xml index 223389c7e51fa..f8c16f3241d92 100644 --- a/flink-staging/flink-hbase/pom.xml +++ b/flink-staging/flink-hbase/pom.xml @@ -51,6 +51,12 @@ under the License. ${project.version} + + org.apache.flink + ${shading-artifact.name} + ${project.version} + + org.apache.flink flink-clients @@ -69,6 +75,12 @@ under the License. flink-hadoop-compatibility ${project.version} test + + + org.apache.flink + flink-shaded-include-yarn + + @@ -81,6 +93,23 @@ under the License. org.mortbay.jetty jetty-util + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-annotations + + + org.apache.hadoop + hadoop-mapreduce-client-core + @@ -97,13 +126,6 @@ under the License. ${hbase.hadoop1.version} - - - - org.apache.hadoop - hadoop-core - - @@ -117,15 +139,8 @@ under the License. ${hbase.hadoop2.version} - - - - org.apache.hadoop - hadoop-common - - - + cdh5.1.3 @@ -145,19 +160,6 @@ under the License. - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - - diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml index 35b6567007a07..76fef0b17ff5a 100644 --- a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml +++ b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml @@ -37,12 +37,6 @@ under the License. - - org.apache.flink - flink-shaded - ${project.version} - - org.apache.flink flink-streaming-core @@ -231,12 +225,25 @@ under the License. - maven-assembly-plugin - - - jar-with-dependencies - - + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + shade-flink + + + + + org.apache.flume:* + com.twitter:hbc-core + com.twitter:joauth + + + + + diff --git a/flink-staging/flink-streaming/flink-streaming-examples/pom.xml b/flink-staging/flink-streaming/flink-streaming-examples/pom.xml index 845ae1bd249b2..0f280dced8b44 100644 --- a/flink-staging/flink-streaming/flink-streaming-examples/pom.xml +++ b/flink-staging/flink-streaming/flink-streaming-examples/pom.xml @@ -117,9 +117,17 @@ under the License. org.apache.maven.plugins maven-jar-plugin - + 2.4 - + + + default + package + + test-jar + + + Iteration @@ -458,18 +466,6 @@ under the License. UTF-8 - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - diff --git a/flink-staging/flink-streaming/flink-streaming-scala/pom.xml b/flink-staging/flink-streaming/flink-streaming-scala/pom.xml index c06fba79d45dd..5a244ad8274b7 100644 --- a/flink-staging/flink-streaming/flink-streaming-scala/pom.xml +++ b/flink-staging/flink-streaming/flink-streaming-scala/pom.xml @@ -69,14 +69,13 @@ under the License. org.ow2.asm asm + ${asm.version} - com.google.guava guava ${guava.version} - provided diff --git a/flink-staging/flink-tachyon/pom.xml b/flink-staging/flink-tachyon/pom.xml index 65aa2955f4dac..b58bc206c6b8c 100644 --- a/flink-staging/flink-tachyon/pom.xml +++ b/flink-staging/flink-tachyon/pom.xml @@ -36,6 +36,11 @@ under the License. This is a Hadoop2 only flink module. --> + + org.apache.flink + ${shading-artifact.name} + ${project.version} + org.apache.flink flink-core @@ -73,16 +78,7 @@ under the License. 7.6.8.v20121106 test - - org.apache.hadoop - hadoop-common - test - - - org.apache.hadoop - hadoop-hdfs - test - + org.apache.hadoop hadoop-hdfs diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml index 9e23e4a2a6d37..1a87496417263 100644 --- a/flink-test-utils/pom.xml +++ b/flink-test-utils/pom.xml @@ -72,12 +72,10 @@ under the License. 4.11 - com.google.guava guava ${guava.version} - provided diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index 95ca77c5a9650..7958b140edacb 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -148,6 +148,13 @@ under the License. joda-time test + + + com.google.guava + guava + ${guava.version} + test + @@ -471,35 +478,4 @@ under the License. - - - - - hadoop-1 - - - - hadoop.profile1 - - - - - - com.google.guava - guava - ${guava.version} - test - - - - - hadoop-2 - - - - !hadoop.profile - - - - diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml index 0cccf3a5e83ae..d93a82dc0cbbf 100644 --- a/flink-yarn-tests/pom.xml +++ b/flink-yarn-tests/pom.xml @@ -43,12 +43,6 @@ under the License. org.apache.flink flink-runtime ${project.version} - - - hadoop-core - org.apache.hadoop - - @@ -64,38 +58,35 @@ under the License. - org.apache.hadoop - hadoop-yarn-client - test - - - - - org.apache.hadoop - hadoop-yarn-common - test + org.apache.flink + ${shading-artifact.name} + ${project.version} - org.apache.hadoop - hadoop-yarn-server-tests - test + org.apache.flink + flink-shaded-include-yarn-tests + ${project.version} + + + asm + asm + + - + - org.apache.hadoop - hadoop-minicluster - test + junit + junit + 4.11 + jar - - org.apache.hadoop - hadoop-mapreduce-client-core - test - + src/main/java + com.google.guava guava ${guava.version} - provided - - org.apache.hadoop - hadoop-yarn-client - - - - org.apache.hadoop - hadoop-common - - - - org.apache.hadoop - hadoop-hdfs - - - - org.apache.hadoop - hadoop-mapreduce-client-core - + diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java b/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java index 5c57292f887fb..16cb3456dad04 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -238,6 +239,15 @@ public int getTaskManagerCount() { @Override public void setShipFiles(List shipFiles) { + File shipFile; + for(Iterator it = shipFiles.iterator(); it.hasNext(); ) { + shipFile = it.next(); + // remove uberjar from ship list (by default everything in the lib/ folder is added to + // the list of files to ship, but we handle the uberjar separately. + if(shipFile.getName().startsWith("flink-dist-") && shipFile.getName().endsWith("jar")) { + it.remove(); + } + } this.shipFiles.addAll(shipFiles); } diff --git a/pom.xml b/pom.xml index 1d09d0279135b..cf042ed0c875a 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ under the License. - flink-shaded + flink-shaded-hadoop flink-core flink-java flink-scala @@ -71,6 +71,7 @@ under the License. UTF-8 UTF-8 + error 1.2.1 2.2.0 2.10.4 @@ -81,13 +82,15 @@ under the License. true log4j-test.properties 1.7.7 - 17.0 + 18.0 2.10.4 2.3.7 2.10 2.0.1 0.3.2 2.5.0 + 0.5.2 + 4.0 @@ -158,14 +161,16 @@ under the License. + - - - - org.ow2.asm - asm - 4.0 - @@ -207,6 +212,12 @@ under the License. commons-cli 1.2 + + + commons-io + commons-io + 2.4 + @@ -310,290 +321,6 @@ under the License. - - - - - org.apache.hadoop - hadoop-core - ${hadoop.version} - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - javax.servlet - servlet-api - - - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - javax.servlet - servlet-api - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - org.apache.hadoop - hadoop-yarn-common - - - com.google.inject.extensions - guice-servlet - - - io.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-client - ${hadoop.version} - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - - - org.apache.hadoop - hadoop-yarn-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-server-tests - test - ${hadoop.version} - - - org.apache.hadoop - hadoop-minicluster - test - ${hadoop.version} - @@ -608,6 +335,7 @@ under the License. ${hadoop-one.version} + flink-shaded-hadoop1 @@ -620,6 +348,7 @@ under the License. ${hadoop-two.version} + flink-shaded-hadoop2 @@ -631,121 +360,14 @@ under the License. !hadoop.profile + + flink-shaded-include-yarn + flink-yarn flink-yarn-tests - - hadoop-2.0.0-alpha - - - hadoop.version - 2.0.0-alpha - - - - 2.2.1 - 0.3.1 - 2.4.1 - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - - org.jboss.netty - netty - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - javax.servlet - servlet-api - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - - - - org.jboss.netty - netty - - - asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jetty-util - - - org.eclipse.jdt - core - - - - - - vendor-repos @@ -896,29 +518,6 @@ under the License. - - - - - org.apache.maven.plugins - maven-surefire-report-plugin - 2.17 - - - - - org.codehaus.mojo - cobertura-maven-plugin - 2.6 - - - html - - - - - - @@ -935,38 +534,6 @@ under the License. - - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - package - - shade - - - false - false - - - org.apache.flink:${project.artifact} - - - - - com.google - org.apache.flink.shaded.com.google - - com.google.protobuf.** - - - - - - - org.apache.rat apache-rat-plugin @@ -1159,6 +726,54 @@ under the License. + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + shade-flink + package + + shade + + + true + false + true + ${project.basedir}/target/dependency-reduced-pom.xml + + + com.google.guava:* + org.ow2.asm:* + + + + + com.google + org.apache.flink.shaded.com.google + + com.google.protobuf.** + com.google.inject.** + + + + org.objectweb.asm + org.apache.flink.shaded.org.objectweb.asm + + + + + + +