diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 1a96c9c5330dd..d7dccade84a0f 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -60,12 +60,10 @@ under the License.
${project.version}
-
com.google.guava
guava
${guava.version}
- provided
@@ -99,8 +97,6 @@ under the License.
commons-io
commons-io
- 2.4
- compile
diff --git a/flink-compiler/pom.xml b/flink-compiler/pom.xml
index 74dae55f9f41c..46d5a9e728bfb 100644
--- a/flink-compiler/pom.xml
+++ b/flink-compiler/pom.xml
@@ -53,12 +53,10 @@ under the License.
${project.version}
-
com.google.guava
guava
${guava.version}
- provided
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 182a77abde741..294a4fe65dda0 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -37,7 +37,7 @@ under the License.
org.apache.flink
- flink-shaded
+ ${shading-artifact.name}
${project.version}
@@ -55,15 +55,13 @@ under the License.
com.twitter
chill_2.10
- 0.5.1
+ ${chill.version}
-
com.google.guava
guava
${guava.version}
- provided
@@ -83,39 +81,4 @@ under the License.
-
-
-
- hadoop-1
-
-
-
- hadoop.profile1
-
-
-
-
-
- org.apache.hadoop
- hadoop-core
-
-
-
-
- hadoop-2
-
-
-
- !hadoop.profile
-
-
-
-
- org.apache.hadoop
- hadoop-common
-
-
-
-
-
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index af3b100e50e23..a39c46ec68142 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -35,18 +35,6 @@ under the License.
-
- org.apache.flink
- flink-shaded
- ${project.version}
-
-
- com.google.guava
- guava
-
-
-
-
org.apache.flink
flink-core
@@ -122,22 +110,6 @@ under the License.
-
- hadoop-1
-
-
-
- hadoop.profile1
-
-
-
- ${hadoop-one.version}
-
-
-
-
-
-
include-yarn
@@ -153,80 +125,6 @@ under the License.
${project.version}
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 2.3
-
-
- package
-
- shade
-
-
- false
- ${project.artifactId}-${project.version}-yarn-uberjar
-
-
- org.apache.flink:flink-java-examples
- org.apache.flink:flink-scala-examples
- org.apache.flink:flink-streaming-examples
-
-
-
-
- org.apache.flink:flink-clients
-
-
- web-docs/**
-
-
-
-
-
-
- reference.conf
-
-
-
-
-
- org.apache.flink.client.FlinkYarnSessionCli
-
-
-
-
-
-
-
-
- maven-assembly-plugin
- 2.4
-
-
-
- yarn-bin
- package
-
- single
-
-
-
- src/main/assemblies/yarn.xml
-
- flink-${project.version}-bin
- false
-
-
-
-
-
-
@@ -361,6 +259,40 @@ under the License.
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 2.3
+
+
+ package
+
+ shade
+
+
+ false
+ false
+ ${project.artifactId}-${project.version}
+
+
+ org.apache.flink:flink-java-examples
+ org.apache.flink:flink-scala-examples
+ org.apache.flink:flink-streaming-examples
+
+
+
+
+ reference.conf
+
+
+
+
+
+
+
+
+
maven-assembly-plugin
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index cd85d3f44d797..64fca1b9f86ec 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -28,29 +28,14 @@ under the License.
true
flink-${project.version}
-
-
-
- lib
- false
- true
- false
- false
- true
-
-
- org.apache.flink:flink-java-examples
- org.apache.flink:flink-scala-examples
- org.apache.flink:flink-streaming-examples
-
-
-
-
-
+
+
+ target/flink-dist-${project.version}.jar
+ lib/
+ 0644
+
+
src/main/resources/flink-conf.yaml
conf
@@ -65,15 +50,17 @@ under the License.
bin
0755
-
+
+
+ src/main/flink-bin/yarn-bin
+ bin
+ 0755
+
src/main/flink-bin/conf
conf
0644
-
- log4j-yarn-session.properties
-
diff --git a/flink-dist/src/main/assemblies/yarn.xml b/flink-dist/src/main/assemblies/yarn.xml
deleted file mode 100644
index 99ebaead36892..0000000000000
--- a/flink-dist/src/main/assemblies/yarn.xml
+++ /dev/null
@@ -1,158 +0,0 @@
-
-
-
- yarn
-
- dir
-
-
- true
- flink-yarn-${project.version}
-
-
-
-
- src/main/flink-bin/yarn-bin
- bin
- 0755
-
-
-
- src/main/flink-bin/
- log
- 0644
-
- **/*
-
-
-
-
-
- src/main/flink-bin/
- ship
- 0644
-
- **/*
-
-
-
-
-
- src/main/flink-bin/
-
- 0644
-
- *.txt
- *.properties
- LICENSE*
- NOTICE*
-
-
-
-
-
- src/main/flink-bin/tools
- tools
- 0644
-
-
- ../flink-clients/src/main/resources/web-docs
- tools
- 0644
-
- *.html
- img/delete-icon.png
- img/GradientBoxes.png
- img/gradient.jpg
-
-
-
-
-
- ../flink-examples/flink-java-examples/target
- examples
- 0644
-
- *.jar
-
-
- flink-java-examples-${project.version}.jar
- original-flink-java-examples-${project.version}.jar
- flink-java-examples-${project.version}-sources.jar
-
-
-
-
-
-
-
-
- src/main/resources/flink-conf.yaml
- conf
- 0644
-
-
- src/main/flink-bin/conf/logback-yarn.xml
- conf
- 0644
-
-
- src/main/flink-bin/conf/log4j.properties
- conf
- 0644
-
-
- src/main/flink-bin/conf/log4j-cli.properties
- conf
- 0644
-
-
- src/main/flink-bin/conf/log4j-yarn-session.properties
- conf
- 0644
-
-
- src/main/flink-bin/conf/logback.xml
- conf
- 0644
-
-
-
-
- target/flink-dist-${project.version}-yarn-uberjar.jar
- lib/
- 0644
-
-
-
- src/main/flink-bin/bin/flink
- bin/
- 0755
-
-
- src/main/flink-bin/bin/config.sh
- bin/
- 0755
-
-
-
diff --git a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
index 3ff2972cd1827..9bf3d0ccae414 100755
--- a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
+++ b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
@@ -52,5 +52,5 @@ log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4
export FLINK_CONF_DIR
-$JAVA_RUN $JVM_ARGS -classpath $CC_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR $log_setting org.apache.flink.client.FlinkYarnSessionCli -ship $bin/../ship/ -j $FLINK_LIB_DIR/*yarn-uberjar.jar $*
+$JAVA_RUN $JVM_ARGS -classpath $CC_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR $log_setting org.apache.flink.client.FlinkYarnSessionCli -ship $bin/../lib/ -j $FLINK_LIB_DIR/flink-dist*.jar $*
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 63d30a475a23e..34fcdd2675117 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -33,9 +33,6 @@ under the License.
flink-java
jar
-
- 0.5.2
-
@@ -43,6 +40,12 @@ under the License.
flink-core
${project.version}
+
+
+ org.apache.flink
+ ${shading-artifact.name}
+ ${project.version}
+
org.apache.avro
@@ -58,6 +61,7 @@ under the License.
org.ow2.asm
asm
+ ${asm.version}
@@ -117,12 +121,10 @@ under the License.
joda-time
-
com.google.guava
guava
${guava.version}
- provided
@@ -150,61 +152,4 @@ under the License.
-
-
-
-
- hadoop-1
-
-
-
- hadoop.profile1
-
-
-
-
-
- org.apache.hadoop
- hadoop-core
-
-
-
-
-
-
-
-
-
-
- hadoop-2
-
-
-
- !hadoop.profile
-
-
-
-
- org.apache.hadoop
- hadoop-common
-
-
-
-
-
-
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
-
-
-
-
-
-
-
-
-
-
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index adc6ab5cb93c1..d84e922382853 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -59,11 +59,6 @@ under the License.
flink-streaming-core
0.9-SNAPSHOT
-
- org.apache.flink
- flink-streaming-connectors
- 0.9-SNAPSHOT
-
org.apache.flink
flink-clients
diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml
index dd3a903a50487..78b99ad181cc1 100644
--- a/flink-quickstart/pom.xml
+++ b/flink-quickstart/pom.xml
@@ -62,8 +62,7 @@ under the License.
${skipTests}
-
-
+
org.apache.maven.plugins
maven-shade-plugin
@@ -73,7 +72,6 @@ under the License.
-
\ No newline at end of file
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 59ec7f09f939c..14e27aa5ea4bf 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -46,11 +46,22 @@ under the License.
flink-java
${project.version}
+
+
+ org.apache.flink
+ ${shading-artifact.name}
+ ${project.version}
+
commons-cli
commons-cli
+
+
+ commons-io
+ commons-io
+
org.eclipse.jetty
@@ -88,12 +99,10 @@ under the License.
1.1
-
com.google.guava
guava
${guava.version}
- provided
@@ -338,43 +347,4 @@ under the License.
-
-
-
-
- hadoop-1
-
-
-
- hadoop.profile1
-
-
-
-
-
- org.apache.hadoop
- hadoop-core
-
-
-
-
- hadoop-2
-
-
-
- !hadoop.profile
-
-
-
-
- org.apache.hadoop
- hadoop-common
-
-
- org.apache.hadoop
- hadoop-hdfs
-
-
-
-
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
index 014cf5791d706..be55e147a9788 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
@@ -232,9 +232,6 @@ object AkkaUtils {
| }
|
| remote {
- | quarantine-systems-for = off
- | gate-invalid-addresses-for = 5 s
- |
| startup-timeout = $startupTimeout
|
| transport-failure-detector{
diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties
index 491b219221c44..76b237e7e92a2 100644
--- a/flink-runtime/src/test/resources/log4j-test.properties
+++ b/flink-runtime/src/test/resources/log4j-test.properties
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-log4j.rootLogger=OFF
+log4j.rootLogger=OFF, console
# -----------------------------------------------------------------------------
# Console (use 'console')
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 3b686abf5312a..32faa6fe743a5 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -75,14 +75,13 @@ under the License.
org.ow2.asm
asm
+ ${asm.version}
-
com.google.guava
guava
${guava.version}
- provided
diff --git a/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml
new file mode 100644
index 0000000000000..1264f74e581a5
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml
@@ -0,0 +1,83 @@
+
+
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ flink-shaded-hadoop
+ 0.9-SNAPSHOT
+ ..
+
+
+ flink-shaded-hadoop1
+ flink-shaded-hadoop1
+
+ jar
+
+
+
+ org.apache.hadoop
+ hadoop-core
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+
+
+
+
diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
new file mode 100644
index 0000000000000..7a441c66ae507
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml
@@ -0,0 +1,195 @@
+
+
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ flink-shaded-hadoop
+ 0.9-SNAPSHOT
+ ..
+
+
+ flink-shaded-hadoop2
+ flink-shaded-hadoop2
+
+ jar
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ javax.servlet
+ servlet-api
+
+
+ javax.servlet.jsp
+ jsp-api
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ javax.servlet
+ servlet-api
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+
+
+ com.google.inject.extensions
+ guice-servlet
+
+
+ io.netty
+ netty
+
+
+
+
+
+
diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
new file mode 100644
index 0000000000000..e271c88eae7e7
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml
@@ -0,0 +1,78 @@
+
+
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ flink-shaded-hadoop
+ 0.9-SNAPSHOT
+ ..
+
+
+ flink-shaded-include-yarn-tests
+ flink-shaded-include-yarn-tests
+
+ jar
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-server-tests
+ ${hadoop.version}
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-server-tests
+ ${hadoop.version}
+ test-jar
+
+
+
+ org.apache.hadoop
+ hadoop-minicluster
+ ${hadoop.version}
+
+
+
+ org.apache.hadoop
+ hadoop-minicluster
+ ${hadoop.version}
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-server-resourcemanager
+ ${hadoop.version}
+
+
+
+
+
diff --git a/flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml b/flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml
new file mode 100644
index 0000000000000..c8e9aa4ea5247
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-include-yarn/pom.xml
@@ -0,0 +1,306 @@
+
+
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ flink-shaded-hadoop
+ 0.9-SNAPSHOT
+ ..
+
+
+ flink-shaded-include-yarn
+ flink-shaded-include-yarn
+
+ jar
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ javax.servlet
+ servlet-api
+
+
+ javax.servlet.jsp
+ jsp-api
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ javax.servlet
+ servlet-api
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+
+
+ com.google.inject.extensions
+ guice-servlet
+
+
+ io.netty
+ netty
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-client
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+
+
+ com.google.inject.extensions
+ guice-servlet
+
+
+ io.netty
+ netty
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+ ${hadoop.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ tomcat
+ jasper-compiler
+
+
+ tomcat
+ jasper-runtime
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jsp-api-2.1
+
+
+ org.mortbay.jetty
+ jsp-2.1
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.eclipse.jdt
+ core
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+
+
+ com.google.inject.extensions
+ guice-servlet
+
+
+ io.netty
+ netty
+
+
+
+
+
diff --git a/flink-shaded/pom.xml b/flink-shaded-hadoop/pom.xml
similarity index 55%
rename from flink-shaded/pom.xml
rename to flink-shaded-hadoop/pom.xml
index 965e6fa953b23..60d81abe2d3fa 100644
--- a/flink-shaded/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -29,29 +29,42 @@ under the License.
..
- flink-shaded
- flink-shaded
+ flink-shaded-hadoop
+ flink-shaded-hadoop
- jar
+ pom
-
-
- com.google.guava
- guava
- ${guava.version}
- compile
-
-
+
+ ${shading-artifact.name}
+
+
+
+
+ include-yarn
+
+
+
+ !hadoop.profile
+
+
+
+ flink-shaded-include-yarn-tests
+
+
+
-
+
org.apache.maven.plugins
maven-shade-plugin
2.3
+ shade-hadoop
package
shade
@@ -60,19 +73,34 @@ under the License.
false
true
${project.basedir}/target/dependency-reduced-pom.xml
-
-
- com.google.guava:guava
-
-
+
+
+ org.slf4j:*
+
+ org/slf4j/impl/StaticLoggerBinder*
+
+
+
+
+
+
+
com.google
- org.apache.flink.shaded.com.google
+ org.apache.flink.hadoop.shaded.com.google
- com.google.protobuf.**
+ com.google.inject.**
+
+ org.objectweb.asm
+ org.apache.flink.hadoop.shaded.org.objectweb.asm
+
+
+ org.jboss.netty
+ org.apache.flink.hadoop.shaded.org.jboss.netty
+
diff --git a/flink-staging/flink-expressions/pom.xml b/flink-staging/flink-expressions/pom.xml
index 800cc2570b3ed..56ed34596348b 100644
--- a/flink-staging/flink-expressions/pom.xml
+++ b/flink-staging/flink-expressions/pom.xml
@@ -38,7 +38,6 @@ under the License.
com.google.guava
guava
${guava.version}
- provided
diff --git a/flink-staging/flink-gelly/pom.xml b/flink-staging/flink-gelly/pom.xml
index 13ed002161e4b..a36ab4bcc715f 100755
--- a/flink-staging/flink-gelly/pom.xml
+++ b/flink-staging/flink-gelly/pom.xml
@@ -51,36 +51,10 @@ under the License.
${project.version}
test
+
+ com.google.guava
+ guava
+ ${guava.version}
+
-
-
-
-
- hadoop-1
-
-
-
- hadoop.profile1
-
-
-
-
-
- com.google.guava
- guava
- ${guava.version}
- provided
-
-
-
-
- hadoop-2
-
-
-
- !hadoop.profile
-
-
-
-
diff --git a/flink-staging/flink-hadoop-compatibility/pom.xml b/flink-staging/flink-hadoop-compatibility/pom.xml
index e39f8b678eee1..d50b98123286e 100644
--- a/flink-staging/flink-hadoop-compatibility/pom.xml
+++ b/flink-staging/flink-hadoop-compatibility/pom.xml
@@ -46,6 +46,13 @@ under the License.
flink-clients
${project.version}
+
+
+ org.apache.flink
+ ${shading-artifact.name}
+ ${project.version}
+
+
org.apache.flink
flink-tests
@@ -59,22 +66,4 @@ under the License.
test
-
-
-
- hadoop-2
-
-
-
- !hadoop.profile
-
-
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
-
-
-
-
diff --git a/flink-staging/flink-hbase/pom.xml b/flink-staging/flink-hbase/pom.xml
index 223389c7e51fa..f8c16f3241d92 100644
--- a/flink-staging/flink-hbase/pom.xml
+++ b/flink-staging/flink-hbase/pom.xml
@@ -51,6 +51,12 @@ under the License.
${project.version}
+
+ org.apache.flink
+ ${shading-artifact.name}
+ ${project.version}
+
+
org.apache.flink
flink-clients
@@ -69,6 +75,12 @@ under the License.
flink-hadoop-compatibility
${project.version}
test
+
+
+ org.apache.flink
+ flink-shaded-include-yarn
+
+
@@ -81,6 +93,23 @@ under the License.
org.mortbay.jetty
jetty-util
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ org.apache.hadoop
+ hadoop-auth
+
+
+ org.apache.hadoop
+ hadoop-annotations
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
@@ -97,13 +126,6 @@ under the License.
${hbase.hadoop1.version}
-
-
-
- org.apache.hadoop
- hadoop-core
-
-
@@ -117,15 +139,8 @@ under the License.
${hbase.hadoop2.version}
-
-
-
- org.apache.hadoop
- hadoop-common
-
-
-
+
cdh5.1.3
@@ -145,19 +160,6 @@ under the License.
-
-
-
- org.apache.hadoop
- hadoop-common
- ${hadoop.version}
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
- ${hadoop.version}
-
-
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
index 35b6567007a07..76fef0b17ff5a 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
@@ -37,12 +37,6 @@ under the License.
-
- org.apache.flink
- flink-shaded
- ${project.version}
-
-
org.apache.flink
flink-streaming-core
@@ -231,12 +225,25 @@ under the License.
- maven-assembly-plugin
-
-
- jar-with-dependencies
-
-
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 2.3
+
+
+ shade-flink
+
+
+
+
+ org.apache.flume:*
+ com.twitter:hbc-core
+ com.twitter:joauth
+
+
+
+
+
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/pom.xml b/flink-staging/flink-streaming/flink-streaming-examples/pom.xml
index 845ae1bd249b2..0f280dced8b44 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-examples/pom.xml
@@ -117,9 +117,17 @@ under the License.
org.apache.maven.plugins
maven-jar-plugin
-
+ 2.4
-
+
+
+ default
+ package
+
+ test-jar
+
+
+
Iteration
@@ -458,18 +466,6 @@ under the License.
UTF-8
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- test-jar
-
-
-
-
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/pom.xml b/flink-staging/flink-streaming/flink-streaming-scala/pom.xml
index c06fba79d45dd..5a244ad8274b7 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-scala/pom.xml
@@ -69,14 +69,13 @@ under the License.
org.ow2.asm
asm
+ ${asm.version}
-
com.google.guava
guava
${guava.version}
- provided
diff --git a/flink-staging/flink-tachyon/pom.xml b/flink-staging/flink-tachyon/pom.xml
index 65aa2955f4dac..b58bc206c6b8c 100644
--- a/flink-staging/flink-tachyon/pom.xml
+++ b/flink-staging/flink-tachyon/pom.xml
@@ -36,6 +36,11 @@ under the License.
This is a Hadoop2 only flink module.
-->
+
+ org.apache.flink
+ ${shading-artifact.name}
+ ${project.version}
+
org.apache.flink
flink-core
@@ -73,16 +78,7 @@ under the License.
7.6.8.v20121106
test
-
- org.apache.hadoop
- hadoop-common
- test
-
-
- org.apache.hadoop
- hadoop-hdfs
- test
-
+
org.apache.hadoop
hadoop-hdfs
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
index 9e23e4a2a6d37..1a87496417263 100644
--- a/flink-test-utils/pom.xml
+++ b/flink-test-utils/pom.xml
@@ -72,12 +72,10 @@ under the License.
4.11
-
com.google.guava
guava
${guava.version}
- provided
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 95ca77c5a9650..7958b140edacb 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -148,6 +148,13 @@ under the License.
joda-time
test
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+ test
+
@@ -471,35 +478,4 @@ under the License.
-
-
-
-
- hadoop-1
-
-
-
- hadoop.profile1
-
-
-
-
-
- com.google.guava
- guava
- ${guava.version}
- test
-
-
-
-
- hadoop-2
-
-
-
- !hadoop.profile
-
-
-
-
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index 0cccf3a5e83ae..d93a82dc0cbbf 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -43,12 +43,6 @@ under the License.
org.apache.flink
flink-runtime
${project.version}
-
-
- hadoop-core
- org.apache.hadoop
-
-
@@ -64,38 +58,35 @@ under the License.
- org.apache.hadoop
- hadoop-yarn-client
- test
-
-
-
-
- org.apache.hadoop
- hadoop-yarn-common
- test
+ org.apache.flink
+ ${shading-artifact.name}
+ ${project.version}
- org.apache.hadoop
- hadoop-yarn-server-tests
- test
+ org.apache.flink
+ flink-shaded-include-yarn-tests
+ ${project.version}
+
+
+ asm
+ asm
+
+
-
+
- org.apache.hadoop
- hadoop-minicluster
- test
+ junit
+ junit
+ 4.11
+ jar
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
- test
-
+ src/main/java
+
com.google.guava
guava
${guava.version}
- provided
-
- org.apache.hadoop
- hadoop-yarn-client
-
-
-
- org.apache.hadoop
- hadoop-common
-
-
-
- org.apache.hadoop
- hadoop-hdfs
-
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
-
+
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java b/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java
index 5c57292f887fb..16cb3456dad04 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java
@@ -26,6 +26,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -238,6 +239,15 @@ public int getTaskManagerCount() {
@Override
public void setShipFiles(List shipFiles) {
+ File shipFile;
+ for(Iterator it = shipFiles.iterator(); it.hasNext(); ) {
+ shipFile = it.next();
+ // remove uberjar from ship list (by default everything in the lib/ folder is added to
+ // the list of files to ship, but we handle the uberjar separately.
+ if(shipFile.getName().startsWith("flink-dist-") && shipFile.getName().endsWith("jar")) {
+ it.remove();
+ }
+ }
this.shipFiles.addAll(shipFiles);
}
diff --git a/pom.xml b/pom.xml
index 1d09d0279135b..cf042ed0c875a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -52,7 +52,7 @@ under the License.
- flink-shaded
+ flink-shaded-hadoop
flink-core
flink-java
flink-scala
@@ -71,6 +71,7 @@ under the License.
UTF-8
UTF-8
+ error
1.2.1
2.2.0
2.10.4
@@ -81,13 +82,15 @@ under the License.
true
log4j-test.properties
1.7.7
- 17.0
+ 18.0
2.10.4
2.3.7
2.10
2.0.1
0.3.2
2.5.0
+ 0.5.2
+ 4.0
@@ -158,14 +161,16 @@ under the License.
+
-
-
-
- org.ow2.asm
- asm
- 4.0
-
@@ -207,6 +212,12 @@ under the License.
commons-cli
1.2
+
+
+ commons-io
+ commons-io
+ 2.4
+
@@ -310,290 +321,6 @@ under the License.
-
-
-
-
- org.apache.hadoop
- hadoop-core
- ${hadoop.version}
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
-
-
-
-
- org.apache.hadoop
- hadoop-common
- ${hadoop.version}
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
- javax.servlet
- servlet-api
-
-
- javax.servlet.jsp
- jsp-api
-
-
-
-
- org.apache.hadoop
- hadoop-hdfs
- ${hadoop.version}
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
- javax.servlet
- servlet-api
-
-
-
-
- org.apache.hadoop
- hadoop-client
- ${hadoop.version}
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
- javax.servlet
- servlet-api
-
-
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
- ${hadoop.version}
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
- org.apache.hadoop
- hadoop-yarn-common
-
-
- com.google.inject.extensions
- guice-servlet
-
-
- io.netty
- netty
-
-
-
-
- org.apache.hadoop
- hadoop-yarn-client
- ${hadoop.version}
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
-
-
- org.apache.hadoop
- hadoop-yarn-common
- ${hadoop.version}
-
-
- org.apache.hadoop
- hadoop-yarn-server-tests
- test
- ${hadoop.version}
-
-
- org.apache.hadoop
- hadoop-minicluster
- test
- ${hadoop.version}
-
@@ -608,6 +335,7 @@ under the License.
${hadoop-one.version}
+ flink-shaded-hadoop1
@@ -620,6 +348,7 @@ under the License.
${hadoop-two.version}
+ flink-shaded-hadoop2
@@ -631,121 +360,14 @@ under the License.
!hadoop.profile
+
+ flink-shaded-include-yarn
+
flink-yarn
flink-yarn-tests
-
- hadoop-2.0.0-alpha
-
-
- hadoop.version
- 2.0.0-alpha
-
-
-
- 2.2.1
- 0.3.1
- 2.4.1
-
-
-
-
- org.apache.hadoop
- hadoop-common
- ${hadoop.version}
-
-
-
- org.jboss.netty
- netty
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- javax.servlet
- servlet-api
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
- ${hadoop.version}
-
-
-
- org.jboss.netty
- netty
-
-
- asm
- asm
-
-
- tomcat
- jasper-compiler
-
-
- tomcat
- jasper-runtime
-
-
- org.mortbay.jetty
- jetty
-
-
- org.mortbay.jetty
- jsp-api-2.1
-
-
- org.mortbay.jetty
- jsp-2.1
-
-
- org.mortbay.jetty
- jetty-util
-
-
- org.eclipse.jdt
- core
-
-
-
-
-
-
vendor-repos
@@ -896,29 +518,6 @@ under the License.
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-report-plugin
- 2.17
-
-
-
-
- org.codehaus.mojo
- cobertura-maven-plugin
- 2.6
-
-
- html
-
-
-
-
-
-
@@ -935,38 +534,6 @@ under the License.
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 2.3
-
-
- package
-
- shade
-
-
- false
- false
-
-
- org.apache.flink:${project.artifact}
-
-
-
-
- com.google
- org.apache.flink.shaded.com.google
-
- com.google.protobuf.**
-
-
-
-
-
-
-
org.apache.rat
apache-rat-plugin
@@ -1159,6 +726,54 @@ under the License.
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 2.3
+
+
+ shade-flink
+ package
+
+ shade
+
+
+ true
+ false
+ true
+ ${project.basedir}/target/dependency-reduced-pom.xml
+
+
+ com.google.guava:*
+ org.ow2.asm:*
+
+
+
+
+ com.google
+ org.apache.flink.shaded.com.google
+
+ com.google.protobuf.**
+ com.google.inject.**
+
+
+
+ org.objectweb.asm
+ org.apache.flink.shaded.org.objectweb.asm
+
+
+
+
+
+
+