Moved WebInfoServer and WebInterfaceServer resources into resource fo…

…lders of respective projects. Jetty uses the jar as base directory.

DEPENDENCIES

@@ -165,7 +165,7 @@ The Apache Flink project packages the following fonts under the
 Open Font License (OFT) - http:https://scripts.sil.org/OFL/
 
  - Font Awesome (http:https://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
- -> fonts in "flink-runtime/resources/web-docs-infoserver/font-awesome/fonts"
+ -> fonts in "flink-runtime/src/main/resources/web-docs-infoserver/font-awesome/fonts"
 
 
 

LICENSE

@@ -293,5 +293,5 @@ The Apache Flink project bundles the following fonts under the
 Open Font License (OFT) - http:https://scripts.sil.org/OFL/
 
  - Font Awesome (http:https://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
- -> fonts in "flink-runtime/resources/web-docs-infoserver/font-awesome/fonts"
+ -> fonts in "flink-runtime/src/main/resources/web-docs-infoserver/font-awesome/fonts"
 

flink-addons/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala

@@ -69,8 +69,6 @@ object ApplicationMaster{
  val taskManagerCount = env.get(Client.ENV_TM_COUNT).toInt
  val slots = env.get(Client.ENV_SLOTS).toInt
  val dynamicPropertiesEncodedString = env.get(Client.ENV_DYNAMIC_PROPERTIES)
- val localWebInterfaceDir =
- s"$currDir/resources/${ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME}"
 
  val appNumber = env.get(Client.ENV_APP_NUMBER).toInt
 
@@ -84,8 +82,8 @@ object ApplicationMaster{
  val jobManagerWebPort = GlobalConfiguration.getInteger(ConfigConstants
  .JOB_MANAGER_WEB_PORT_KEY, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT)
 
- generateConfigurationFile(currDir, ownHostname, jobManagerPort, localWebInterfaceDir,
- jobManagerWebPort, logDirs, slots, taskManagerCount, dynamicPropertiesEncodedString)
+ generateConfigurationFile(currDir, ownHostname, jobManagerPort, jobManagerWebPort,
+ logDirs, slots, taskManagerCount, dynamicPropertiesEncodedString)
 
  val (system, actor) = startJobManager(currDir)
 
@@ -114,8 +112,9 @@ object ApplicationMaster{
  }
 
  def generateConfigurationFile(currDir: String, ownHostname: String, jobManagerPort: Int,
- localWebInterfaceDir: String, jobManagerWebPort: Int, logDirs:
- String, slots: Int, taskManagerCount: Int, dynamicPropertiesEncodedString: String): Unit = {
+ jobManagerWebPort: Int, logDirs: String, slots: Int,
+ taskManagerCount: Int, dynamicPropertiesEncodedString: String)
+ : Unit = {
  val output = new PrintWriter(new BufferedWriter(new FileWriter(s"$currDir/flink-conf-modified" +
  s".yaml")))
 
@@ -128,7 +127,6 @@ object ApplicationMaster{
 
  output.println(s"${ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY}: $ownHostname")
  output.println(s"${ConfigConstants.JOB_MANAGER_IPC_PORT_KEY}: $jobManagerPort")
- output.println(s"${ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY}: $localWebInterfaceDir")
  output.println(s"${ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY}: $logDirs")
  output.println(s"${ConfigConstants.JOB_MANAGER_WEB_PORT_KEY}: $jobManagerWebPort")
 
@@ -151,8 +149,6 @@ object ApplicationMaster{
  }
 
  def startJobManager(currDir: String): (ActorSystem, ActorRef) = {
-// Utils.copyJarContents(s"resources/${ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME}",
-// classOf[YarnMaster].getProtectionDomain.getCodeSource.getLocation.getPath)
 
  val pathToConfig = s"$currDir/flink-conf.modified.yaml"
  val args = Array[String]("--configDir", pathToConfig)

flink-addons/pom.xml

@@ -34,11 +34,11 @@ under the License.
  <packaging>pom</packaging>
 
  <modules>
- <!--<module>flink-avro</module>-->
- <!--<module>flink-jdbc</module>-->
- <!--<module>flink-spargel</module>-->
- <!--<module>flink-hadoop-compatibility</module>-->
- <!--<module>flink-streaming</module>-->
+ <module>flink-avro</module>
+ <module>flink-jdbc</module>
+ <module>flink-spargel</module>
+ <module>flink-hadoop-compatibility</module>
+ <module>flink-streaming</module>
  <module>flink-hbase</module>
  </modules>
 

flink-clients/src/main/java/org/apache/flink/client/web/WebInterfaceServer.java

@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.URL;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -45,6 +46,9 @@
  * configures an embedded jetty server.
  */
 public class WebInterfaceServer {
+
+ private static final String WEB_ROOT_DIR = "web-docs";
+
  /**
  * The log for this class.
  */
@@ -81,20 +85,15 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
  // get base path of Flink installation
  String basePath = nepheleConfig.getString(ConfigConstants.FLINK_BASE_DIR_PATH_KEY,"");
 
- File webDir;
  File tmpDir;
  File uploadDir;
  File planDumpDir;
 
- String webDirPath = config.getString(ConfigConstants.WEB_ROOT_PATH_KEY,
- ConfigConstants.DEFAULT_WEB_ROOT_DIR);
-
- if(webDirPath.startsWith("/")) {
- // absolute path
- webDir = new File(webDirPath);
- } else {
- // path relative to base dir
- webDir = new File(basePath+"/"+webDirPath);
+ URL webRootDir = this.getClass().getClassLoader().getResource(WEB_ROOT_DIR);
+
+ if(webRootDir == null){
+ throw new FileNotFoundException("Cannot start web interface server because the web " +
+ "root dir " + WEB_ROOT_DIR + " is not included in the jar.");
  }
 
  String tmpDirPath = config.getString(ConfigConstants.WEB_TMP_DIR_KEY,
@@ -131,7 +130,8 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
  }
 
  if (LOG.isInfoEnabled()) {
- LOG.info("Setting up web frontend server, using web-root directory '" + webDir.getAbsolutePath() + "'.");
+ LOG.info("Setting up web frontend server, using web-root directory '" +
+ webRootDir.toExternalForm() + "'.");
  LOG.info("Web frontend server will store temporary files in '" + tmpDir.getAbsolutePath()
  + "', uploaded jobs in '" + uploadDir.getAbsolutePath() + "', plan-json-dumps in '"
  + planDumpDir.getAbsolutePath() + "'.");
@@ -144,12 +144,6 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
 
  server = new Server(port);
 
- // ensure that the directory with the web documents exists
- if (!webDir.exists()) {
- throw new FileNotFoundException("The directory containing the web documents does not exist: "
- + webDir.getAbsolutePath());
- }
-
  // ensure, that all the directories exist
  checkAndCreateDirectories(tmpDir, true);
  checkAndCreateDirectories(uploadDir, true);
@@ -178,7 +172,7 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
  // ----- the handler serving all the static files -----
  ResourceHandler resourceHandler = new ResourceHandler();
  resourceHandler.setDirectoriesListed(false);
- resourceHandler.setResourceBase(webDir.getAbsolutePath());
+ resourceHandler.setResourceBase(webRootDir.toExternalForm());
 
  // ----- add the handlers to the list handler -----
  HandlerList handlers = new HandlerList();

flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java

@@ -249,11 +249,6 @@ public final class ConfigConstants {
  */
  public static final String JOB_MANAGER_WEB_PORT_KEY = "jobmanager.web.port";
 
- /**
- * The parameter defining the directory containing the web documents for the jobmanager web frontend.
- */
- public static final String JOB_MANAGER_WEB_ROOT_PATH_KEY = "jobmanager.web.rootpath";
-
  /**
  * The config parameter defining the path to the htaccess file protecting the web frontend.
  */
@@ -274,11 +269,6 @@ public final class ConfigConstants {
  */
  public static final String WEB_FRONTEND_PORT_KEY = "webclient.port";
 
- /**
- * The config parameter defining the directory containing the web documents.
- */
- public static final String WEB_ROOT_PATH_KEY = "webclient.rootpath";
-
  /**
  * The config parameter defining the temporary data directory for the web client.
  */
@@ -299,9 +289,6 @@ public final class ConfigConstants {
  */
  public static final String WEB_ACCESS_FILE_KEY = "webclient.access";
 
- // ----------------------------- YARN Client ----------------------------
-
- public static final String YARN_AM_PRC_PORT = "yarn.am.rpc.port";
 
  // ------------------------------ AKKA ------------------------------------
 
@@ -549,16 +536,6 @@ public final class ConfigConstants {
  */
  public static final int DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT = 8081;
 
- /**
- * The default directory name of the info server
- */
- public static final String DEFAULT_JOB_MANAGER_WEB_PATH_NAME = "web-docs-infoserver";
-
- /**
- * The default path of the directory for info server containing the web documents.
- */
- public static final String DEFAULT_JOB_MANAGER_WEB_ROOT_PATH = "./resources/"+DEFAULT_JOB_MANAGER_WEB_PATH_NAME+"/";
-
  /**
  * The default number of archived jobs for the jobmanager
  */
@@ -572,11 +549,6 @@ public final class ConfigConstants {
  */
  public static final int DEFAULT_WEBCLIENT_PORT = 8080;
 
- /**
- * The default path of the directory containing the web documents.
- */
- public static final String DEFAULT_WEB_ROOT_DIR = "./resources/web-docs/";
-
  /**
  * The default directory to store temporary objects (e.g. during file uploads).
  */
@@ -599,10 +571,6 @@ public final class ConfigConstants {
  */
  public static final String DEFAULT_WEB_ACCESS_FILE_PATH = null;
 
- // ----------------------------- YARN ----------------------------
-
- public static final int DEFAULT_YARN_AM_RPC_PORT = 10245;
-
  // ------------------------------ Akka Values ------------------------------
 
  public static String DEFAULT_AKKA_TRANSPORT_HEARTBEAT_INTERVAL = "1000 ms";
@@ -635,19 +603,6 @@ public final class ConfigConstants {
  */
  public static final String LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER = "localinstancemanager.numtaskmanager";
 
-
- // ----------------------------- Deprecated --------------------------------
-
- /**
- * The default definition for an instance type, if no other configuration is provided.
- */
- public static final String DEFAULT_INSTANCE_TYPE = "default,1,1,1,1,0"; // minimalistic instance type until "cloud" model is fully removed.
-
- /**
- * The default index for the default instance type.
- */
- public static final int DEFAULT_DEFAULT_INSTANCE_TYPE_INDEX = 1;
-
  // ------------------------------------------------------------------------
 
  /**

flink-dist/pom.xml

@@ -31,7 +31,7 @@ under the License.
 
  <artifactId>flink-dist</artifactId>
  <name>flink-dist</name>
- <packaging>pom</packaging>
+ <packaging>jar</packaging>
 
  <dependencies>
  <!-- BINARIES -->
@@ -162,35 +162,71 @@ under the License.
  <build>
  <plugins>
  <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4</version><!--$NO-MVN-MAN-VER$-->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
  <executions>
- <!-- Uber-jar -->
  <execution>
- <id>uber-jar</id>
  <phase>package</phase>
  <goals>
- <goal>single</goal>
+ <goal>shade</goal>
  </goals>
- <configuration>
- <archiverConfig>
- <!-- https://jira.codehaus.org/browse/MASSEMBLY-449 -->
- <fileMode>420</fileMode> <!-- 420(dec) = 644(oct) -->
- <directoryMode>493</directoryMode> <!-- 493(dec) = 755(oct) -->
- <defaultDirectoryMode>493</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifest>
- <mainClass>org.apache.flink.yarn.Client</mainClass>
- <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
- <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
- </manifest>
- </archive>
- <descriptors>
- <descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>
- </descriptors>
+ <configuration combine.self="override">
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <finalName>${project.artifactId}-${project.version}-yarn-uberjar</finalName>
+ <artifactSet>
+ <excludes>
+ <exclude>org.apache.flink:flink-java-examples</exclude>
+ <exclude>org.apache.flink:flink-scala-examples</exclude>
+ <exclude>org.apache.flink:flink-streaming-examples</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Main-Class>org.apache.flink.yarn.Client</Main-Class>
+ </manifestEntries>
+ </transformer>
+ </transformers>
  </configuration>
  </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <!--&lt;!&ndash; Uber-jar &ndash;&gt;-->
+ <!--<execution>-->
+ <!--<id>uber-jar</id>-->
+ <!--<phase>package</phase>-->
+ <!--<goals>-->
+ <!--<goal>single</goal>-->
+ <!--</goals>-->
+ <!--<configuration>-->
+ <!--<archiverConfig>-->
+ <!--&lt;!&ndash; https://jira.codehaus.org/browse/MASSEMBLY-449 &ndash;&gt;-->
+ <!--<fileMode>420</fileMode> &lt;!&ndash; 420(dec) = 644(oct) &ndash;&gt;-->
+ <!--<directoryMode>493</directoryMode> &lt;!&ndash; 493(dec) = 755(oct) &ndash;&gt;-->
+ <!--<defaultDirectoryMode>493</defaultDirectoryMode>-->
+ <!--</archiverConfig>-->
+ <!--<archive>-->
+ <!--<manifest>-->
+ <!--<mainClass>org.apache.flink.yarn.Client</mainClass>-->
+ <!--<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>-->
+ <!--<addDefaultImplementationEntries>true</addDefaultImplementationEntries>-->
+ <!--</manifest>-->
+ <!--</archive>-->
+ <!--<descriptors>-->
+ <!--<descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>-->
+ <!--</descriptors>-->
+ <!--</configuration>-->
+ <!--</execution>-->
 
  <!-- yarn bin directory -->
  <execution>

flink-dist/src/main/assemblies/bin.xml

@@ -103,26 +103,6 @@ under the License.
  <fileMode>0644</fileMode>
  </fileSet>
 
- <fileSet>
- <!-- copy files for compiler web frontend -->
- <directory>../flink-clients/resources</directory>
- <outputDirectory>resources</outputDirectory>
- <fileMode>0644</fileMode>
- <excludes>
- <exclude>*etc/users</exclude>
- </excludes>
- </fileSet>
-
- <fileSet>
- <!-- copy files for Jobmanager web frontend -->
- <directory>../flink-runtime/resources</directory>
- <outputDirectory>resources</outputDirectory>
- <fileMode>0644</fileMode>
- <excludes>
- <exclude>*etc/users</exclude>
- </excludes>
- </fileSet>
-
  <!-- copy the tools -->
  <fileSet>
  <directory>src/main/flink-bin/tools</directory>

flink-dist/src/main/assemblies/yarn-uberjar.xml

@@ -54,16 +54,6 @@ under the License.
  </files>
 
  <fileSets>
- <!-- copy files for Jobmanager web frontend -->
- <fileSet>
- <directory>../nephele/nephele-server/resources</directory>
- <outputDirectory>resources</outputDirectory>
- <fileMode>0644</fileMode>
- <excludes>
- <exclude>*etc/users</exclude>
- </excludes>
- </fileSet>
-
  <!-- copy files for Jobmanager web frontend -->
  <fileSet>
  <directory>../flink-runtime/resources</directory>