[FLINK-17460][orc][parquet] Create sql-jars for parquet and orc

This closes apache#11946
derekm · May 6, 2020 · d40cf24 · d40cf24
1 parent a8cee7b
commit d40cf24
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 1 deletion.
diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md
@@ -59,6 +59,8 @@ The following tables list all available connectors and formats. Their mutual com
 | CSV (for Kafka) | `flink-csv` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/{{site.version}}/flink-csv-{{site.version}}-sql-jar.jar) |
 | JSON | `flink-json` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/{{site.version}}/flink-json-{{site.version}}-sql-jar.jar) |
 | Apache Avro | `flink-avro` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar) |
+| Apache ORC | `flink-orc` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-orc{{site.scala_version_suffix}}/{{site.version}}/flink-orc{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
+| Apache Parquet | `flink-parquet` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-parquet{{site.scala_version_suffix}}/{{site.version}}/flink-parquet{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
 
 {% else %}
 

diff --git a/flink-formats/flink-orc/pom.xml b/flink-formats/flink-orc/pom.xml
@@ -84,6 +84,14 @@ under the License.
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-hdfs</artifactId>
  </exclusion>
+ <exclusion>
+ <groupId>javax.activation</groupId>
+ <artifactId>javax.activation-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
  </exclusions>
  </dependency>
 
@@ -169,7 +177,25 @@ under the License.
  </execution>
  </executions>
  </plugin>
+
+ <!-- build a jar-with-dependencies SQL Client uber jars -->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
-
 </project>
diff --git a/flink-formats/flink-orc/src/main/resources/META-INF/services/NOTICE b/flink-formats/flink-orc/src/main/resources/META-INF/services/NOTICE
@@ -0,0 +1,18 @@
+flink-orc
+Copyright 2014-2020 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0. (https://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- org.apache.orc:orc-core:1.5.6
+- org.apache.orc:orc-shims:1.5.6
+- org.apache.hive:hive-storage-api:2.6.0
+- io.airlift:aircompressor:0.10
+- commons-lang:commons-lang:2.6
+
+This project bundles the following dependencies under the BSD license.
+See bundled license files for details.
+
+- com.google.protobuf:protobuf-java:2.5.0
diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml
@@ -87,6 +87,12 @@ under the License.
  <groupId>org.apache.parquet</groupId>
  <artifactId>parquet-hadoop</artifactId>
  <version>${flink.format.parquet.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
  </dependency>
 
  <!-- Hadoop is needed by Parquet -->
@@ -200,6 +206,25 @@ under the License.
  </execution>
  </executions>
  </plugin>
+
+ <!-- build a jar-with-dependencies SQL Client uber jars -->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
  </build>
 </project>
diff --git a/flink-formats/flink-parquet/src/main/resources/META-INF/services/NOTICE b/flink-formats/flink-parquet/src/main/resources/META-INF/services/NOTICE
@@ -0,0 +1,21 @@
+flink-orc
+Copyright 2014-2020 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0. (https://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- org.apache.parquet:parquet-hadoop:1.10.0
+- org.apache.parquet:parquet-column:1.10.0
+- org.apache.parquet:parquet-common:1.10.0
+- org.apache.parquet:parquet-encoding:1.10.0
+- org.apache.parquet:parquet-format:2.4.0
+- org.apache.parquet:parquet-jackson:1.10.0
+- org.apache.parquet:parquet-avro:1.10.0
+- org.codehaus.jackson:jackson-mapper-asl:1.9.13
+- org.codehaus.jackson:jackson-core-asl:1.9.13
+- org.apache.commons:commons-compress:1.20
+- org.apache.avro:avro:1.8.2
+- commons-pool:commons-pool:1.6
+- commons-codec:commons-codec:1.10