Skip to content

Commit

Permalink
[FLINK-1605] Bundle all hadoop dependencies and shade guava away
Browse files Browse the repository at this point in the history 
This closes apache#454
  • Loading branch information
@rmetzger
rmetzger committed Mar 11, 2015
1 parent 2522f02 commit 84e76f4
Show file tree
Hide file tree
Showing 39 changed files with 1,030 additions and 1,140 deletions.
4 changes: 0 additions & 4 deletions flink-clients/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,10 @@ under the License.
<version>${project.version}</version>
</dependency>

<!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
Expand Down Expand Up @@ -99,8 +97,6 @@ under the License.
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
<scope>compile</scope>
</dependency>

<dependency>
Expand Down
2 changes: 0 additions & 2 deletions flink-compiler/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,10 @@ under the License.
<version>${project.version}</version>
</dependency>

<!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>

Expand Down
41 changes: 2 additions & 39 deletions flink-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ under the License.
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded</artifactId>
<artifactId>${shading-artifact.name}</artifactId>
<version>${project.version}</version>
</dependency>

Expand All @@ -55,15 +55,13 @@ under the License.
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_2.10</artifactId>
<version>0.5.1</version>
<version>${chill.version}</version>
</dependency>

<!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>

Expand All @@ -83,39 +81,4 @@ under the License.
</plugins>
</build>

<!-- See main pom.xml for explanation of profiles -->
<profiles>
<profile>
<id>hadoop-1</id>
<activation>
<property>
<!-- Please do not remove the 'hadoop1' comment. See ./tools/generate_specific_pom.sh -->
<!--hadoop1--><name>hadoop.profile</name><value>1</value>
</property>
</activation>
<dependencies>
<!-- "Old" Hadoop = MapReduce v1 -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
</dependency>
</dependencies>
</profile>
<profile>
<id>hadoop-2</id>
<activation>
<property>
<!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
<!--hadoop2--><name>!hadoop.profile</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>

</project>
136 changes: 34 additions & 102 deletions flink-dist/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,6 @@ under the License.

<dependencies>
<!-- BINARIES -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
Expand Down Expand Up @@ -122,22 +110,6 @@ under the License.

<!-- See main pom.xml for explanation of profiles -->
<profiles>
<profile>
<id>hadoop-1</id>
<activation>
<property>
<!-- Please do not remove the 'hadoop1' comment. See ./tools/generate_specific_pom.sh -->
<!--hadoop1--><name>hadoop.profile</name><value>1</value>
</property>
</activation>
<properties>
<hadoop.version>${hadoop-one.version}</hadoop.version>
</properties>
<dependencies>
<!-- No extra dependencies: flink-hbase is currently not compatible with Hadoop v1 -->
</dependencies>
</profile>

<profile>
<id>include-yarn</id>
<activation>
Expand All @@ -153,80 +125,6 @@ under the License.
<version>${project.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<!--Build uber jar-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration combine.self="override">
<shadedArtifactAttached>false</shadedArtifactAttached>
<finalName>${project.artifactId}-${project.version}-yarn-uberjar</finalName>
<artifactSet>
<excludes>
<exclude>org.apache.flink:flink-java-examples</exclude>
<exclude>org.apache.flink:flink-scala-examples</exclude>
<exclude>org.apache.flink:flink-streaming-examples</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>org.apache.flink:flink-clients</artifact>
<excludes>
<exclude>
web-docs/**
</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<!-- The service transformer is needed to merge META-INF/services files -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>org.apache.flink.client.FlinkYarnSessionCli</Main-Class>
</manifestEntries>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version><!--$NO-MVN-MAN-VER$-->
<executions>
<!-- yarn bin directory -->
<execution>
<id>yarn-bin</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assemblies/yarn.xml</descriptor>
</descriptors>
<finalName>flink-${project.version}-bin</finalName>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

<profile>
Expand Down Expand Up @@ -361,6 +259,40 @@ under the License.

<build>
<plugins>
<plugin>
<!--Build uber jar-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration combine.self="override">
<createDependencyReducedPom>false</createDependencyReducedPom>
<shadedArtifactAttached>false</shadedArtifactAttached>
<finalName>${project.artifactId}-${project.version}</finalName>
<artifactSet>
<excludes>
<exclude>org.apache.flink:flink-java-examples</exclude>
<exclude>org.apache.flink:flink-scala-examples</exclude>
<exclude>org.apache.flink:flink-streaming-examples</exclude>
</excludes>
</artifactSet>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<!-- The service transformer is needed to merge META-INF/services files -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
Expand Down
39 changes: 13 additions & 26 deletions flink-dist/src/main/assemblies/bin.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,29 +28,14 @@ under the License.
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>flink-${project.version}</baseDirectory>

<!-- copy all jar files and their dependencies -->
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
<useTransitiveDependencies>true</useTransitiveDependencies>
<useProjectArtifact>false</useProjectArtifact>
<useProjectAttachments>false</useProjectAttachments>
<useTransitiveFiltering>true</useTransitiveFiltering>

<excludes>
<exclude>org.apache.flink:flink-java-examples</exclude>
<exclude>org.apache.flink:flink-scala-examples</exclude>
<exclude>org.apache.flink:flink-streaming-examples</exclude>
<!--
<exclude>**/*javadoc*</exclude>
<exclude>**/*sources*</exclude>
-->
</excludes>
</dependencySet>
</dependencySets>

<files>
<!-- copy fat jar -->
<file>
<source>target/flink-dist-${project.version}.jar</source>
<outputDirectory>lib/</outputDirectory>
<fileMode>0644</fileMode>
</file>

<file>
<source>src/main/resources/flink-conf.yaml</source>
<outputDirectory>conf</outputDirectory>
Expand All @@ -65,15 +50,17 @@ under the License.
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>

<fileSet>
<!-- copy yarn start scripts -->
<directory>src/main/flink-bin/yarn-bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<!-- copy default configuration -->
<directory>src/main/flink-bin/conf</directory>
<outputDirectory>conf</outputDirectory>
<fileMode>0644</fileMode>
<excludes>
<exclude>log4j-yarn-session.properties</exclude>
</excludes>
</fileSet>

<fileSet>
Expand Down
Loading

0 comments on commit 84e76f4

Please sign in to comment.