[FLINK-18173][build] Bundle flink-csv and flink-json jars in lib

This closes apache#12527
deepfield · Jun 15, 2020 · 2b2b643 · 2b2b643
1 parent 5e43f01
commit 2b2b643
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 28 deletions.
diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md
@@ -54,11 +54,9 @@ The following tables list all available connectors and formats. Their mutual com
 | Name | Maven dependency | SQL Client JAR |
 | :------------------------- | :--------------------------- | :--------------------- |
 | Old CSV (for files) | Built-in | Built-in |
-| CSV (for Kafka) | `flink-csv` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/{{site.version}}/flink-csv-{{site.version}}-sql-jar.jar) |
-| JSON | `flink-json` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/{{site.version}}/flink-json-{{site.version}}-sql-jar.jar) |
+| CSV (for Kafka) | `flink-csv` | Built-in  |
+| JSON | `flink-json` | Built-in  |
 | Apache Avro | `flink-avro` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar) |
-| Apache ORC | `flink-orc` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-orc{{site.scala_version_suffix}}/{{site.version}}/flink-orc{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
-| Apache Parquet | `flink-parquet` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-parquet{{site.scala_version_suffix}}/{{site.version}}/flink-parquet{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
 
 {% else %}
 

diff --git a/docs/dev/table/connect.zh.md b/docs/dev/table/connect.zh.md
@@ -54,11 +54,9 @@ The following tables list all available connectors and formats. Their mutual com
 | Name | Maven dependency | SQL Client JAR |
 | :------------------------- | :--------------------------- | :--------------------- |
 | Old CSV (for files) | Built-in | Built-in |
-| CSV (for Kafka) | `flink-csv` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/{{site.version}}/flink-csv-{{site.version}}-sql-jar.jar) |
-| JSON | `flink-json` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/{{site.version}}/flink-json-{{site.version}}-sql-jar.jar) |
+| CSV (for Kafka) | `flink-csv` | Built-in  |
+| JSON | `flink-json` | Built-in  |
 | Apache Avro | `flink-avro` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar) |
-| Apache ORC | `flink-orc` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-orc{{site.scala_version_suffix}}/{{site.version}}/flink-orc{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
-| Apache Parquet | `flink-parquet` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-parquet{{site.scala_version_suffix}}/{{site.version}}/flink-parquet{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) |
 
 {% else %}
 

diff --git a/docs/dev/table/connectors/formats/avro.md b/docs/dev/table/connectors/formats/avro.md
@@ -38,7 +38,8 @@ In order to setup the Avro format, the following table provides dependency infor
 
 <div class="codetabs" markdown="1">
 <div data-lang="SQL Client JAR" markdown="1">
-Avro format is part of the binary distribution, but requires additional [Hadoop dependency]({% link ops/deployment/hadoop.md %}) for cluster execution.
+You can download flink-avro from [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar),
+and requires additional [Hadoop dependency]({% link ops/deployment/hadoop.md %}) for cluster execution.
 </div>
 <div data-lang="Maven dependency" markdown="1">
 {% highlight xml %}

diff --git a/docs/dev/table/connectors/formats/avro.zh.md b/docs/dev/table/connectors/formats/avro.zh.md
@@ -38,7 +38,8 @@ In order to setup the Avro format, the following table provides dependency infor
 
 <div class="codetabs" markdown="1">
 <div data-lang="SQL Client JAR" markdown="1">
-Avro format is part of the binary distribution, but requires additional [Hadoop dependency]({% link ops/deployment/hadoop.zh.md %}) for cluster execution.
+You can download flink-avro from [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar),
+and requires additional [Hadoop dependency]({% link ops/deployment/hadoop.md %}) for cluster execution.
 </div>
 <div data-lang="Maven dependency" markdown="1">
 {% highlight xml %}

diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
@@ -84,6 +84,21 @@ under the License.
  <fileMode>0644</fileMode>
  </file>
 
+ <!-- Built-in formats -->
+ <file>
+ <source>../flink-formats/flink-csv/target/flink-csv-${project.version}.jar</source>
+ <outputDirectory>lib/</outputDirectory>
+ <destName>flink-csv-${project.version}.jar</destName>
+ <fileMode>0644</fileMode>
+ </file>
+
+ <file>
+ <source>../flink-formats/flink-json/target/flink-json-${project.version}.jar</source>
+ <outputDirectory>lib/</outputDirectory>
+ <destName>flink-json-${project.version}.jar</destName>
+ <fileMode>0644</fileMode>
+ </file>
+
  <!-- copy the config file -->
  <file>
  <source>src/main/resources/flink-conf.yaml</source>

diff --git a/flink-end-to-end-tests/flink-sql-client-test/pom.xml b/flink-end-to-end-tests/flink-sql-client-test/pom.xml
@@ -160,20 +160,6 @@ under the License.
  <classifier>sql-jar</classifier>
  <type>jar</type>
  </artifactItem>
- <artifactItem>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- <version>${project.version}</version>
- <classifier>sql-jar</classifier>
- <type>jar</type>
- </artifactItem>
- <artifactItem>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-csv</artifactId>
- <version>${project.version}</version>
- <classifier>sql-jar</classifier>
- <type>jar</type>
- </artifactItem>
  <artifactItem>
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-sql-connector-kafka-0.10_${scala.binary.version}</artifactId>

diff --git a/flink-end-to-end-tests/test-scripts/test_sql_client.sh b/flink-end-to-end-tests/test-scripts/test_sql_client.sh
@@ -142,7 +142,6 @@ start_taskmanagers 2
 
 echo "Testing SQL statements..."
 
-JSON_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "json" )
 KAFKA_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "kafka_" )
 ELASTICSEARCH_SQL_JAR=$(find "$SQL_JARS_DIR" | grep "elasticsearch$ELASTICSEARCH_VERSION" )
 
@@ -224,7 +223,6 @@ EOF
 
 JOB_ID=$($FLINK_DIR/bin/sql-client.sh embedded \
  --jar $KAFKA_SQL_JAR \
- --jar $JSON_SQL_JAR \
  --jar $ELASTICSEARCH_SQL_JAR \
  --jar $SQL_TOOLBOX_JAR \
  --environment $SQL_CONF \
@@ -253,7 +251,6 @@ EOF
 
 JOB_ID=$($FLINK_DIR/bin/sql-client.sh embedded \
  --jar $KAFKA_SQL_JAR \
- --jar $JSON_SQL_JAR \
  --jar $ELASTICSEARCH_SQL_JAR \
  --jar $SQL_TOOLBOX_JAR \
  --environment $SQL_CONF \
@@ -286,7 +283,6 @@ EOF
 
 JOB_ID=$($FLINK_DIR/bin/sql-client.sh embedded \
  --jar $KAFKA_SQL_JAR \
- --jar $JSON_SQL_JAR \
  --jar $ELASTICSEARCH_SQL_JAR \
  --jar $SQL_TOOLBOX_JAR \
  --environment $SQL_CONF \