[FLINK-4118] Update docker image to 1.0.3 and remove unneeded deps

Some of the changes include:

- Remove unneeded dependencies (nano, wget)
- Remove apt lists to reduce image size
- Reduce number of layers on the docker image (best docker practice)
- Remove useless variables and base the code in generic ones e.g.
FLINK_HOME
- Change the default JDK from oracle to openjdk-8-jre-headless, based on
two reasons:

1. You cannot legally repackage the oracle jdk in docker images
2. The open-jdk headless is more appropriate for a server image (no GUI stuff)

- Return port assignation to the standard FLINK one:

Variable: docker-flink -> flink

taskmanager.rpc.port: 6121 -> 6122
taskmanager.data.port: 6122 -> 6121
jobmanager.web.port: 8080 -> 8081

This closes apache#2176

flink-contrib/docker-flink/flink/Dockerfile → flink-contrib/docker-flink/Dockerfile

@@ -16,28 +16,25 @@
 # limitations under the License.
 ################################################################################
 
-FROM base
+FROM java:8-jre-alpine
 
-#add passless key to ssh
-RUN ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
-RUN cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/*
+# Install requirements
+RUN apk add --no-cache bash curl snappy supervisor
 
-##Flink 1.0.2 Installation
-###Download:
-RUN mkdir ~/downloads && cd ~/downloads && \
- wget -q -O - http:https://mirror.switch.ch/mirror/apache/dist/flink/flink-1.0.2/flink-1.0.2-bin-hadoop27-scala_2.11.tgz| tar -zxvf - -C /usr/local/
-RUN cd /usr/local && ln -s ./flink-1.0.2 flink
+# Configure supervisor
+ADD supervisor.conf /etc/supervisor/
 
+# Install Flink
+ARG FLINK_VERSION=1.0.3
+ARG HADOOP_VERSION=27
+ARG SCALA_VERSION=2.11
+
+RUN curl -s $(curl -s https://www.apache.org/dyn/closer.cgi\?as_json\=1 | awk '/preferred/ {gsub(/"/,""); print $2}')flink/flink-${FLINK_VERSION}/flink-${FLINK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala_${SCALA_VERSION}.tgz | tar xvz -C /usr/local/
+RUN ln -s /usr/local/flink-$FLINK_VERSION /usr/local/flink
 ENV FLINK_HOME /usr/local/flink
 ENV PATH $PATH:$FLINK_HOME/bin
 
-#config files (template)
-ADD conf/flink-conf.yaml /usr/local/flink/conf/
-
-ADD config-flink.sh /usr/local/flink/bin/
-RUN chmod +x /usr/local/flink/bin/config-flink.sh
-
-EXPOSE 6123
-EXPOSE 22
-
-CMD ["/usr/local/flink/bin/config-flink.sh", "taskmanager"]
+# Configure container
+ADD docker-entrypoint.sh $FLINK_HOME/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["sh", "-c"]

flink-contrib/docker-flink/README.md

@@ -1,80 +1,81 @@
-#Apache Flink cluster deployment on Docker using Docker-Compose
+Apache Flink cluster deployment on docker using docker-compose
 
-##Installation
-###Install Docker
+# Installation
 
+Install the most recent stable version of docker
 https://docs.docker.com/installation/
 
-if you have issues with Docker-Compose versions incompatible with your version of Docker try
+Install the most recent stable version of docker-compose
+https://docs.docker.com/compose/install/
 
-`curl -sSL https://get.docker.com/ubuntu/ | sudo sh`
+# Build
 
-###Install Docker-Compose
+Images are based on the official Java Alpine (OpenJDK 8) image and run
+supervisord to stay alive when running containers. If you want to build the
+flink image run:
 
-```
-curl -L https://github.com/docker/compose/releases/download/1.1.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
+ sh build.sh
 
-chmod +x /usr/local/bin/docker-compose
-```
-
-###Get the repo
-
-###Build the images
+or
 
-Images are based on Ubuntu Trusty 14.04 and run Supervisord to stay alive when running containers.
+ docker build -t flink .
 
-The base image installs Oracle Java JDK 1.7 and SSH client & server. You can change the SSH password there or add your own key and adjust SSH config.
+If you want to build the container for a specific version of flink/hadoop/scala
+you can configure it in the respective args:
 
-- Run `./build.sh`
+ docker build --build-arg FLINK_VERSION=1.0.3 --build-arg HADOOP_VERSION=26 --build-arg SCALA_VERSION=2.10 -t "flink:1.0.3-hadoop2.6-scala_2.10" flink
 
-###Deploy
+# Deploy
 
 - Deploy cluster and see config/setup log output (best run in a screen session)
 
-`docker-compose up`
+  docker-compose up
 
 - Deploy as a daemon (and return)
 
-`docker-compose up -d`
+  docker-compose up -d
 
 - Scale the cluster up or down to *N* TaskManagers
 
-`docker-compose scale taskmanager=<N>`
+  docker-compose scale taskmanager=<N>
 
-- Access the JobManager node with SSH (exposed on Port 220)
+- Access the Job Manager container
 
-`ssh root@localhost -p 220`
+  docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) /bin/sh
 
-or on Mac OS X with boot2docker
+- Kill the cluster
 
-`ssh root@$(boot2docker ip) -p 220`
+  docker-compose kill
 
-The password is 'secret'
+- Upload jar to the cluster
 
-- Kill the cluster
+  docker cp <your_jar> $(docker ps --filter name=flink_jobmanager --format={{.ID}}):/<your_path>
 
-`docker-compose kill`
+- Copy file to all the nodes in the cluster
 
-- Upload a jar to the cluster
-
-`scp -P 220 <your_jar> root@localhost:/<your_path>`
+  for i in $(docker ps --filter name=flink --format={{.ID}}); do
+  docker cp <your_file> $i:/<your_path>
+  done
 
 - Run a topology
 
-`ssh -p 220 root@localhost /usr/local/flink/bin/flink run -c <your_class> <your_jar> <your_params>`
+From the jobmanager:
+
+  docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) flink run -m <jobmanager:port> -c <your_class> <your_jar> <your_params>
+
+If you have a local flink installation:
+
+  $FLINK_HOME/bin/flink run -m <jobmanager:port> <your_jar>
 
 or
 
-ssh to the job manager and run the topology from there.
+  $FLINK_HOME/bin/flink run -m <jobmanager:port> -c <your_class> <your_jar> <your_params>
 
-###Ports
+### Ports
 
-- The Web Dashboard is on port `48080`
 - The Web Client is on port `48081`
 - JobManager RPC port `6123` (default, not exposed to host)
-- TaskManagers RPC port `6121` (default, not exposed to host)
-- TaskManagers Data port `6122` (default, not exposed to host)
-- JobManager SSH `220`
-- TaskManagers SSH: randomly assigned port, check wih `docker ps`
+- TaskManagers RPC port `6122` (default, not exposed to host)
+- TaskManagers Data port `6121` (default, not exposed to host)
 
 Edit the `docker-compose.yml` file to edit port settings.

flink-contrib/docker-flink/build.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 ################################################################################
 # Licensed to the Apache Software Foundation (ASF) under one
@@ -18,14 +18,4 @@
 # limitations under the License.
 ################################################################################
 
-
-#Kill cluster and remove all containers
-docker-compose kill
-#docker rm $(docker ps -a -q)
-
-#make sure the config file script is executable
-chmod +x flink/config-flink.sh
-
-#rebuild images
-docker build -t="base" base
-docker build -t="flink" flink
+docker build -t "flink" .

flink-contrib/docker-flink/docker-compose.yml

@@ -16,22 +16,22 @@
 # limitations under the License.
 ################################################################################
 
-jobmanager:
+version: "2"
+services:
+ jobmanager:
  image: flink
  ports:
- - "48080:8080"
- - "48081:8081"
- - "220:22"
- expose:
- - "6123"
- - "22"
- command: /usr/local/flink/bin/config-flink.sh jobmanager
-taskmanager:
+ - "48081:8081"
+ command: jobmanager
+ volumes:
+ - conf:/usr/local/flink/conf
+
+ taskmanager:
  image: flink
- ports:
-  - "22"
- expose:
-  - "6121"
-  - "6122"
- links:
-  - jobmanager:jobmanager
+ depends_on:
+ - jobmanager
+ command: taskmanager
+ volumes_from:
+ - jobmanager:ro
+volumes:
+ conf:

flink-contrib/docker-flink/flink/conf/flink-conf.yaml → flink-contrib/docker-flink/docker-entrypoint.sh

@@ -1,3 +1,5 @@
+#!/bin/sh
+
 ################################################################################
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements. See the NOTICE file
@@ -16,21 +18,20 @@
 # limitations under the License.
 ################################################################################
 
-jobmanager.rpc.address: %jobmanager%
-jobmanager.rpc.port: 6123
-jobmanager.heap.mb: 128
-
-taskmanager.rpc.port: 6121
-taskmanager.data.port: 6122
-taskmanager.heap.mb: 256
-taskmanager.numberOfTaskSlots: %nb_slots%
-
-parallelization.degree.default: %parallelism%
-
-env.java.home: /usr/java/default
+if [ "$1" = "jobmanager" ]; then
+ echo "Starting Job Manager"
+ sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -i`/g" $FLINK_HOME/conf/flink-conf.yaml
+ sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c ^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
+ $FLINK_HOME/bin/jobmanager.sh start cluster
+ echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
+ supervisord -c /etc/supervisor/supervisor.conf
 
-#==============================================================================
-# Web Frontend
-#==============================================================================
+elif [ "$1" = "taskmanager" ]; then
+ echo "Starting Task Manager"
+ $FLINK_HOME/bin/taskmanager.sh start
+ echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
+ supervisord -c /etc/supervisor/supervisor.conf
 
-jobmanager.web.port: 8080
+else
+ $@
+fi