diff --git a/docs/_config.yml b/docs/_config.yml index aea41bd472930..5a83c9a95e4e4 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -34,6 +34,8 @@ version_title: "1.8-SNAPSHOT" # Branch on Github for this version github_branch: "master" +# Plain Scala version is needed for e.g. the Gradle quickstart. +scala_version: "2.11" # This suffix is appended to the Scala-dependent Maven artifact names scala_version_suffix: "_2.11" diff --git a/docs/dev/projectsetup/java_api_quickstart.md b/docs/dev/projectsetup/java_api_quickstart.md index 03d75c130720b..91ef70e9326ce 100644 --- a/docs/dev/projectsetup/java_api_quickstart.md +++ b/docs/dev/projectsetup/java_api_quickstart.md @@ -26,14 +26,24 @@ under the License. * This will be replaced by the TOC {:toc} -Start working on your Flink Java program in a few simple steps. +## Build Tools -## Requirements +Flink projects can be built with different build tools. +In order to get started quickly, Flink provides project templates for the following build tools: + +- [Maven](#maven) +- [Gradle](#gradle) + +These templates help you to set up the project structure and to create the initial build files. + +## Maven + +### Requirements The only requirements are working __Maven 3.0.4__ (or higher) and __Java 8.x__ installations. -## Create Project +### Create Project Use one of the following commands to __create a project__: @@ -69,7 +79,7 @@ Use one of the following commands to __create a project__: {% endunless %} -## Inspect Project +### Inspect Project There will be a new directory in your working directory. If you've used the _curl_ approach, the directory is called `quickstart`. Otherwise, @@ -108,7 +118,7 @@ In Eclipse, choose box: `-Xmx800m`. In IntelliJ IDEA recommended way to change JVM options is from the `Help | Edit Custom VM Options` menu. See [this article](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) for details. -## Build Project +### Build Project If you want to __build/package your project__, go to your project directory and run the '`mvn clean package`' command. @@ -116,8 +126,212 @@ You will __find a JAR file__ that contains your application, plus connectors and that you may have added as dependencies to the application: `target/-.jar`. __Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, -we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink -can run time application from the JAR file without additionally specifying the main class. +we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, Flink +can run the application from the JAR file without additionally specifying the main class. + +## Gradle + +### Requirements + +The only requirements are working __Gradle 3.x__ (or higher) and __Java 8.x__ installations. + +### Create Project + +Use one of the following commands to __create a project__: + + +
+
+ + +
+ +
+ {% highlight gradle %} +buildscript { + repositories { + jcenter() // this applies only to the Gradle 'Shadow' plugin + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4' + } +} + +plugins { + id 'java' + id 'application' + // shadow plugin to produce fat JARs + id 'com.github.johnrengelman.shadow' version '2.0.4' +} + + +// artifact properties +group = 'org.myorg.quickstart' +version = '0.1-SNAPSHOT' +mainClassName = 'org.myorg.quickstart.StreamingJob' +description = """Flink Quickstart Job""" + +ext { + javaVersion = '1.8' + flinkVersion = '{{ site.version }}' + scalaBinaryVersion = '{{ site.scala_version }}' + slf4jVersion = '1.7.7' + log4jVersion = '1.2.17' +} + + +sourceCompatibility = javaVersion +targetCompatibility = javaVersion +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"] + +task wrapper(type: Wrapper) { + gradleVersion = '3.1' +} + +// declare where to find the dependencies of your project +repositories { + mavenCentral() + maven { url "https://repository.apache.org/content/repositories/snapshots/" } +} + +// NOTE: We cannot use "compileOnly" or "shadow" configurations since then we could not run code +// in the IDE or with "gradle run". We also cannot exclude transitive dependencies from the +// shadowJar yet (see https://github.com/johnrengelman/shadow/issues/159). +// -> Explicitly define the // libraries we want to be included in the "flinkShadowJar" configuration! +configurations { + flinkShadowJar // dependencies which go into the shadowJar + + // always exclude these (also from transitive dependencies) since they are provided by Flink + flinkShadowJar.exclude group: 'org.apache.flink', module: 'force-shading' + flinkShadowJar.exclude group: 'com.google.code.findbugs', module: 'jsr305' + flinkShadowJar.exclude group: 'org.slf4j' + flinkShadowJar.exclude group: 'log4j' +} + +// declare the dependencies for your production and test code +dependencies { + // -------------------------------------------------------------- + // Compile-time dependencies that should NOT be part of the + // shadow jar and are provided in the lib folder of Flink + // -------------------------------------------------------------- + compile "org.apache.flink:flink-java:${flinkVersion}" + compile "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}" + + // -------------------------------------------------------------- + // Dependencies that should be part of the shadow jar, e.g. + // connectors. These must be in the flinkShadowJar configuration! + // -------------------------------------------------------------- + //flinkShadowJar "org.apache.flink:flink-connector-kafka-0.11_${scalaBinaryVersion}:${flinkVersion}" + + compile "log4j:log4j:${log4jVersion}" + compile "org.slf4j:slf4j-log4j12:${slf4jVersion}" + + // Add test dependencies here. + // testCompile "junit:junit:4.12" +} + +// make compileOnly dependencies available for tests: +sourceSets { + main.compileClasspath += configurations.flinkShadowJar + main.runtimeClasspath += configurations.flinkShadowJar + + test.compileClasspath += configurations.flinkShadowJar + test.runtimeClasspath += configurations.flinkShadowJar + + javadoc.classpath += configurations.flinkShadowJar +} + +run.classpath = sourceSets.main.runtimeClasspath + +jar { + manifest { + attributes 'Built-By': System.getProperty('user.name'), + 'Build-Jdk': System.getProperty('java.version') + } +} + +shadowJar { + configurations = [project.configurations.flinkShadowJar] +} + {% endhighlight %} +
+
+ {% highlight gradle %} +rootProject.name = 'quickstart' + {% endhighlight %} +
+
+
+ +
+ {% highlight bash %} + bash -c "$(curl https://flink.apache.org/q/gradle-quickstart.sh)" -- {{site.version}} {{site.scala_version}} + {% endhighlight %} + This allows you to name your newly created project. It will interactively ask + you for the project name, organization (also used for the package name), project version, + Scala and Flink version. +
+
+ +### Inspect Project + +There will be a new directory in your working directory based on the +project name you provided, e.g. for `quickstart`: + +{% highlight bash %} +$ tree quickstart/ +quickstart/ +├── README +├── build.gradle +├── settings.gradle +└── src + └── main + ├── java + │   └── org + │   └── myorg + │   └── quickstart + │   ├── BatchJob.java + │   └── StreamingJob.java + └── resources + └── log4j.properties +{% endhighlight %} + +The sample project is a __Gradle project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. + +We recommend you __import this project into your IDE__ to develop and +test it. IntelliJ IDEA supports Gradle projects after installing the `Gradle` plugin. +Eclipse does so via the [Eclipse Buildship](https://projects.eclipse.org/projects/tools.buildship) plugin +(make sure to specify a Gradle version >= 3.0 in the last step of the import wizard; the `shadow` plugin requires it). +You may also use [Gradle's IDE integration](https://docs.gradle.org/current/userguide/userguide.html#ide-integration) +to create project files from Gradle. + + +*A note to Mac OS X users*: The default JVM heapsize for Java may be too +small for Flink. You have to manually increase it. In Eclipse, choose +`Run Configurations -> Arguments` and write into the `VM Arguments` +box: `-Xmx800m`. + +### Build Project + +If you want to __build/package your project__, go to your project directory and +run the '`gradle clean shadowJar`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencies to the application: `build/libs/--all.jar`. + +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClassName` setting in the `build.gradle` file accordingly. That way, Flink +can run the application from the JAR file without additionally specifying the main class. ## Next Steps