common_mesos_docker.sh

#!/usr/bin/env bash
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http:https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

set -o pipefail

source "$(dirname "$0")"/common.sh
source "$(dirname "$0")"/common_docker.sh

MAX_RETRY_SECONDS=120
IMAGE_BUILD_RETRIES=5
NODENAME=${NODENAME:-`hostname -f`}
export MESOS_AGENT_CPU=1

echo "End-to-end directory $END_TO_END_DIR"

start_time=$(date +%s)

# make sure we stop our cluster at the end
function cluster_shutdown {
  docker exec mesos-master bash -c "chmod -R ogu+rw $FLINK_LOG_DIR/ ${TEST_DATA_DIR}"
  docker-compose -f $END_TO_END_DIR/test-scripts/docker-mesos-cluster/docker-compose.yml down
}
on_exit cluster_shutdown

function start_flink_cluster_with_mesos() {
    echo "Starting Flink on Mesos cluster"
    if ! retry_times $IMAGE_BUILD_RETRIES 0 build_image; then
        echo "ERROR: Could not build mesos image. Aborting..."
        exit 1
    fi
    # build docker image with java and mesos
    build_image

    # we need to export the MVN_REPO location so that mesos can access the files referenced in HADOOP_CLASSPATH
    export MVN_REPO=`mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout`

    # start mesos cluster
    docker-compose -f $END_TO_END_DIR/test-scripts/docker-mesos-cluster/docker-compose.yml up -d

    # wait for the Mesos master and slave set up
    start_time=$(date +%s)
    wait_rest_endpoint_up "http:https://${NODENAME}:5050/slaves" "Mesos" "\{\"slaves\":\[.+\].*\}"

    # perform health checks
    containers_health_check "mesos-master" "mesos-slave"

    set_config_key "jobmanager.rpc.address" "mesos-master"
    set_config_key "rest.address" "mesos-master"

    docker exec --env HADOOP_CLASSPATH=$HADOOP_CLASSPATH -itd mesos-master bash -c "${FLINK_DIR}/bin/mesos-appmaster.sh -Dmesos.master=mesos-master:5050"

    wait_rest_endpoint_up "http:https://${NODENAME}:8081/taskmanagers" "Dispatcher" "\{\"taskmanagers\":\[.*\]\}"
    return 0
}

function build_image() {
    echo "Building Mesos Docker container"
    docker build -f $END_TO_END_DIR/test-scripts/docker-mesos-cluster/Dockerfile \
        -t flink/docker-mesos-cluster:latest \
        $END_TO_END_DIR/test-scripts/docker-mesos-cluster/
}

function wait_job_terminal_state_mesos {
  local job=$1
  local expected_terminal_state=$2
  wait_job_terminal_state $1 $2 "mesos-appmaster"
}

function wait_num_of_occurence_in_logs_mesos() {
    local text=$1
    local number=$2
    wait_num_of_occurence_in_logs $1 $2 "mesos-appmaster"
}