[FLINK-9822] Add Dockerfile for StandaloneJobClusterEntryPoint image

This commit adds a Dockerfile for a standalone job cluster image. The image
contains the Flink distribution and a specified user code jar. The entrypoint
will start the StandaloneJobClusterEntryPoint with the provided job classname.

This closes apache#6319.

flink-container/docker/Dockerfile

@@ -0,0 +1,50 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+FROM java:8-jre-alpine
+
+# Install requirements
+RUN apk add --no-cache bash snappy
+
+# Flink environment variables
+ENV FLINK_INSTALL_PATH=/opt
+ENV FLINK_HOME $FLINK_INSTALL_PATH/flink
+ENV FLINK_LIB_DIR $FLINK_HOME/lib
+ENV PATH $PATH:$FLINK_HOME/bin
+
+# flink-dist can point to a directory or a tarball on the local system
+ARG flink_dist=NOT_SET
+ARG job_jar=NOT_SET
+
+# Install build dependencies and flink
+ADD $flink_dist $FLINK_INSTALL_PATH
+ADD $job_jar $FLINK_INSTALL_PATH/job.jar
+
+RUN set -x && \
+ ln -s $FLINK_INSTALL_PATH/flink-* $FLINK_HOME && \
+ ln -s $FLINK_INSTALL_PATH/job.jar $FLINK_LIB_DIR && \
+ addgroup -S flink && adduser -D -S -H -G flink -h $FLINK_HOME flink && \
+ chown -R flink:flink $FLINK_INSTALL_PATH/flink-* && \
+ chown -h flink:flink $FLINK_HOME
+
+COPY docker-entrypoint.sh /
+
+USER flink
+EXPOSE 8081 6123
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["--help"]

flink-container/docker/README.md

@@ -0,0 +1,40 @@
+# Apache Flink job cluster deployment on docker using docker-compose
+
+## Installation
+
+Install the most recent stable version of docker
+https://docs.docker.com/installation/
+
+## Build
+
+Images are based on the official Java Alpine (OpenJDK 8) image. If you want to
+build the flink image run:
+
+ build.sh --from-local-dist --job-jar /path/to/job/jar/job.jar --image-name flink:job
+
+If you want to build the container for a specific version of flink/hadoop/scala
+you can configure it in the respective args:
+
+ docker build --build-arg FLINK_VERSION=1.6.0 --build-arg HADOOP_VERSION=28 --build-arg SCALA_VERSION=2.11 -t "flink:1.6.0-hadoop2.8-scala_2.11" flink
+
+## Deploy
+
+- Deploy cluster and see config/setup log output (best run in a screen session)
+
+  docker-compose up
+
+- Deploy as a daemon (and return)
+
+  docker-compose up -d
+
+- Scale the cluster up or down to *N* TaskManagers
+
+  docker-compose scale taskmanager=<N>
+
+- Access the Job Manager container
+
+  docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) /bin/sh
+
+- Kill the cluster
+
+  docker-compose kill

flink-container/docker/build.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+usage() {
+ cat <<HERE
+Usage:
+ build.sh --job-jar <path-to-job-jar> --from-local-dist [--image-name <image>]
+ build.sh --job-jar <path-to-job-jar> --from-release --flink-version <x.x.x> --hadoop-version <x.x> --scala-version <x.xx> [--image-name <image>]
+ build.sh --help
+
+ If the --image-name flag is not used the built image name will be 'flink'.
+HERE
+ exit 1
+}
+
+while [[ $# -ge 1 ]]
+do
+key="$1"
+ case $key in
+ --job-jar)
+ JOB_JAR_PATH="$2"
+ ;;
+ --from-local-dist)
+ FROM_LOCAL="true"
+ ;;
+ --from-release)
+ FROM_RELEASE="true"
+ ;;
+ --image-name)
+ IMAGE_NAME="$2"
+ shift
+ ;;
+ --flink-version)
+ FLINK_VERSION="$2"
+ shift
+ ;;
+ --hadoop-version)
+ HADOOP_VERSION="$(echo "$2" | sed 's/\.//')"
+ shift
+ ;;
+ --scala-version)
+ SCALA_VERSION="$2"
+ shift
+ ;;
+ --kubernetes-certificates)
+ CERTIFICATES_DIR="$2"
+ shift
+ ;;
+ --help)
+ usage
+ ;;
+ *)
+ # unknown option
+ ;;
+ esac
+ shift
+done
+
+IMAGE_NAME=${IMAGE_NAME:-flink-job}
+
+# TMPDIR must be contained within the working directory so it is part of the
+# Docker context. (i.e. it can't be mktemp'd in /tmp)
+TMPDIR=_TMP_
+
+cleanup() {
+ rm -rf "${TMPDIR}"
+}
+trap cleanup EXIT
+
+mkdir -p "${TMPDIR}"
+
+JOB_JAR_TARGET="${TMPDIR}/job.jar"
+cp ${JOB_JAR_PATH} ${JOB_JAR_TARGET}
+
+if [ -n "${FROM_RELEASE}" ]; then
+
+ [[ -n "${FLINK_VERSION}" ]] && [[ -n "${HADOOP_VERSION}" ]] && [[ -n "${SCALA_VERSION}" ]] || usage
+
+ FLINK_DIST_FILE_NAME="flink-${FLINK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala_${SCALA_VERSION}.tgz"
+ CURL_OUTPUT="${TMPDIR}/${FLINK_DIST_FILE_NAME}"
+
+ echo "Downloading ${FLINK_DIST_FILE_NAME} from ${FLINK_BASE_URL}"
+ curl -# "https://archive.apache.org/dist/flink/flink-${FLINK_VERSION}/${FLINK_DIST_FILE_NAME}" --output ${CURL_OUTPUT}
+
+ FLINK_DIST="${CURL_OUTPUT}"
+
+elif [ -n "${FROM_LOCAL}" ]; then
+
+ DIST_DIR="../../flink-dist/target/flink-*-bin"
+ FLINK_DIST="${TMPDIR}/flink.tgz"
+ echo "Using flink dist: ${DIST_DIR}"
+ tar -C ${DIST_DIR} -cvzf "${FLINK_DIST}" .
+
+else
+
+ usage
+
+fi
+
+docker build --build-arg flink_dist="${FLINK_DIST}" --build-arg job_jar="${JOB_JAR_TARGET}" -t "${IMAGE_NAME}" .

flink-container/docker/docker-compose.yml

@@ -0,0 +1,31 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Set the FLINK_DOCKER_IMAGE_NAME environment variable to override the image name to use
+
+version: "2.1"
+services:
+ job-cluster:
+ image: ${FLINK_DOCKER_IMAGE_NAME:-flink-job}
+ ports:
+ - "8081:8081"
+ command: job-cluster --job-classname ${FLINK_JOB} -Djobmanager.rpc.address=job-cluster
+
+ taskmanager:
+ image: ${FLINK_DOCKER_IMAGE_NAME:-flink-job}
+ command: task-manager -Djobmanager.rpc.address=job-cluster

flink-container/docker/docker-entrypoint.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+### If unspecified, the hostname of the container is taken as the JobManager address
+FLINK_HOME=${FLINK_HOME:-"/opt/flink/bin"}
+
+JOB_CLUSTER="job-cluster"
+TASK_MANAGER="task-manager"
+
+CMD="$1"
+shift;
+
+if [ "${CMD}" == "--help" -o "${CMD}" == "-h" ]; then
+ echo "Usage: $(basename $0) (${JOB_CLUSTER}|${TASK_MANAGER})"
+ exit 0
+elif [ "${CMD}" == "${JOB_CLUSTER}" -o "${CMD}" == "${TASK_MANAGER}" ]; then
+ echo "Starting the ${CMD}"
+ echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
+
+ if [ "${CMD}" == "${TASK_MANAGER}" ]; then
+ exec $FLINK_HOME/bin/taskmanager.sh start-foreground "$@"
+ else
+ exec $FLINK_HOME/bin/standalone-job.sh start-foreground "$@"
+ fi
+fi
+
+exec "$@"