Skip to content

Commit

Permalink
- Allow dependencies to retain guava
Browse files Browse the repository at this point in the history 
- Clean dependencies in flink-streaming connectors
  • Loading branch information
@StephanEwen
StephanEwen committed Sep 30, 2014
1 parent 626d6b7 commit 719e288
Show file tree
Hide file tree
Showing 22 changed files with 432 additions and 90 deletions.
1 change: 1 addition & 0 deletions flink-addons/flink-avro/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ under the License.
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>provided</scope>
</dependency>

Expand Down
34 changes: 34 additions & 0 deletions flink-addons/flink-hadoop-compatibility/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,40 @@ under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
Expand Down
18 changes: 10 additions & 8 deletions ...lity/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.flink.hadoopcompatibility.mapred.record;

import java.util.Collections;
import java.util.List;

import org.apache.flink.api.common.operators.Operator;
Expand All @@ -31,9 +32,6 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;

/**
* The HadoopDataSink is a generic wrapper for all Hadoop OutputFormats.
*
Expand All @@ -54,7 +52,7 @@ public class HadoopDataSink<K,V> extends GenericDataSink implements Validatable
private JobConf jobConf;

public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, Operator<Record> input, FlinkTypeConverter<K,V> conv, Class<K> keyClass, Class<V> valueClass) {
this(hadoopFormat, jobConf, name, ImmutableList.<Operator<Record>>of(input), conv, keyClass, valueClass);
this(hadoopFormat, jobConf, name, Collections.<Operator<Record>>singletonList(input), conv, keyClass, valueClass);
}

public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, Operator<Record> input, Class<K> keyClass, Class<V> valueClass) {
Expand All @@ -74,8 +72,11 @@ public HadoopDataSink(OutputFormat<K,V> hadoopFormat, Operator<Record> input, Cl
@SuppressWarnings("deprecation")
public HadoopDataSink(OutputFormat<K,V> hadoopFormat, JobConf jobConf, String name, List<Operator<Record>> input, FlinkTypeConverter<K,V> conv, Class<K> keyClass, Class<V> valueClass) {
super(new HadoopRecordOutputFormat<K,V>(hadoopFormat, jobConf, conv),input, name);
Preconditions.checkNotNull(hadoopFormat);
Preconditions.checkNotNull(jobConf);

if (hadoopFormat == null || jobConf == null) {
throw new NullPointerException();
}

this.name = name;
this.jobConf = jobConf;
jobConf.setOutputKeyClass(keyClass);
Expand All @@ -101,7 +102,8 @@ public JobConf getJobConf() {
@Override
public void check() {
// see for more details https://github.com/stratosphere/stratosphere/pull/531
Preconditions.checkNotNull(FileOutputFormat.getOutputPath(jobConf), "The HadoopDataSink currently expects a correct outputPath.");
if (FileOutputFormat.getOutputPath(jobConf) == null) {
throw new NullPointerException("The HadoopDataSink currently expects a correct outputPath.");
}
}

}
14 changes: 5 additions & 9 deletions ...ty/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* limitations under the License.
*/


package org.apache.flink.hadoopcompatibility.mapred.record;


Expand All @@ -26,10 +25,6 @@
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;

import com.google.common.base.Preconditions;



/**
* The HadoopDataSource is a generic wrapper for all Hadoop InputFormats.
*
Expand All @@ -44,7 +39,6 @@
* The HadoopDataSource provides two different standard converters:
* * WritableWrapperConverter: Converts Hadoop Types to a record that contains a WritableComparableWrapper (key) and a WritableWrapper
* * DefaultHadoopTypeConverter: Converts the standard hadoop types (longWritable, Text) to Flinks's {@link org.apache.flink.types.Value} types.
*
*/
public class HadoopDataSource<K,V> extends GenericDataSource<HadoopRecordInputFormat<K,V>> {

Expand All @@ -61,9 +55,11 @@ public class HadoopDataSource<K,V> extends GenericDataSource<HadoopRecordInputFo
*/
public HadoopDataSource(InputFormat<K,V> hadoopFormat, JobConf jobConf, String name, HadoopTypeConverter<K,V> conv) {
super(new HadoopRecordInputFormat<K,V>(hadoopFormat, jobConf, conv),name);
Preconditions.checkNotNull(hadoopFormat);
Preconditions.checkNotNull(jobConf);
Preconditions.checkNotNull(conv);

if (hadoopFormat == null || jobConf == null || conv == null) {
throw new NullPointerException();
}

this.name = name;
this.jobConf = jobConf;
}
Expand Down
37 changes: 28 additions & 9 deletions flink-addons/flink-hbase/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,34 @@ under the License.
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
Expand All @@ -104,13 +132,4 @@ under the License.
to use profiles See ticket https://issues.apache.org/jira/browse/HADOOP-8009
for description of hadoop-clients -->

<reporting>
<plugins>
</plugins>
</reporting>

<build>
<plugins>
</plugins>
</build>
</project>
12 changes: 11 additions & 1 deletion flink-addons/flink-streaming/flink-streaming-connectors/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -72,9 +76,15 @@ under the License.
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>1.5.0</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>hbc-core</artifactId>
<version>2.2.0</version>
Expand Down
144 changes: 144 additions & 0 deletions flink-addons/flink-yarn/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,28 +51,172 @@ under the License.
<version>${project.version}</version>
</dependency>

<!-- guava needs to be in "provided" scope, so to not re-export the dependency -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
Loading

0 comments on commit 719e288

Please sign in to comment.