[FLINK-11086] Replace flink-shaded-hadoop-2 dependency by vanilla Had…

…oop dependency
spotify · May 13, 2020 · e1e7d7f · e1e7d7f
1 parent 6a6a439
commit e1e7d7f
Show file tree

Hide file tree

Showing 24 changed files with 390 additions and 168 deletions.
diff --git a/flink-connectors/flink-connector-filesystem/pom.xml b/flink-connectors/flink-connector-filesystem/pom.xml
@@ -51,8 +51,20 @@ under the License.
  </dependency>
 
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
  <scope>provided</scope>
  </dependency>
 

diff --git a/flink-connectors/flink-connector-hive/pom.xml b/flink-connectors/flink-connector-hive/pom.xml
@@ -41,6 +41,91 @@ under the License.
  <derby.version>10.10.2.0</derby.version>
  </properties>
 
+ <!-- Overwrite hadoop dependency management from flink-parent to use locally defined Hadoop version -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
  <dependencies>
 
  <!-- core dependencies -->
@@ -126,12 +211,16 @@ under the License.
  thus override the default hadoop version from 2.4.1 to 2.7.5
  -->
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <version>${hivemetastore.hadoop.version}-${flink.shaded.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
  <scope>provided</scope>
  </dependency>
 
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
  <!-- Hive dependencies -->
  <!-- Note: Hive published jars do not have proper dependencies declared.
  We need to push for HIVE-16391 (https://issues.apache.org/jira/browse/HIVE-16391) to resolve this problem. -->

diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -61,8 +61,14 @@ under the License.
  </dependency>
 
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
  <scope>provided</scope>
  </dependency>
 

diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml
@@ -101,8 +101,8 @@ under the License.
  </dependency>
 
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
  <scope>provided</scope>
  </dependency>
 

diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
@@ -82,8 +82,14 @@ under the License.
  </dependency>
 
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
  <scope>provided</scope>
  </dependency>
 

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
@@ -137,8 +137,8 @@ under the License.
  <version>${project.version}</version>
  <exclusions>
  <exclusion>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
  </exclusion>
  </exclusions>
  </dependency>
@@ -460,50 +460,6 @@ under the License.
  </dependencies>
  </profile>
 
- <profile>
- <!-- Copies that shaded Hadoop uber jar to the dist folder. -->
- <id>include-hadoop</id>
- <activation>
- <property>
- <name>include-hadoop</name>
- </property>
- </activation>
- <dependencies>
- <!--
- The Hadoop 2 Uber jar should not go into the Flink dist jar, but
- sit next to it. Hence, we set it to 'provided' here.
- -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>hadoop</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>src/main/assemblies/hadoop.xml</descriptor>
- </descriptors>
- <finalName>flink-${project.version}-bin</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
- </profile>
  <profile>
  <!-- Creates/Removes the 'build-target' symlink in the root directory (only Unix systems) -->
  <id>symlink-build-target</id>

diff --git a/flink-dist/src/main/assemblies/hadoop.xml b/flink-dist/src/main/assemblies/hadoop.xml
diff --git a/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml b/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml
@@ -47,8 +47,13 @@
  <version>${project.version}</version>
  </dependency>
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
  <scope>provided</scope>
  <exclusions>
  <!-- Needed for proper dependency convergence -->

diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
@@ -255,6 +255,21 @@ under the License.
  </execution>
  </executions>
  </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dependency-convergence</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
  </plugins>
 
  <pluginManagement>

diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml
@@ -43,10 +43,23 @@ under the License.
  <!-- pulling in Hadoop by default -->
 
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
  <optional>true</optional>
  </dependency>
+
 
  <!-- for the behavior test suite -->
  <dependency>

diff --git a/flink-filesystems/flink-mapr-fs/pom.xml b/flink-filesystems/flink-mapr-fs/pom.xml
@@ -47,8 +47,14 @@ under the License.
  </dependency>
 
  <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
  <optional>true</optional>
  </dependency>
 

diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
@@ -32,6 +32,17 @@ under the License.
 
  <packaging>jar</packaging>
 
+ <!-- Override the flink-parent dependencyManagement definition for hadoop-common to ensure
+ ${fs.hadoopshaded.version} is used for this file system -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
  <dependencies>
 
  <!-- Flink's file system abstraction (compiled against, not bundled) -->