#!/usr/bin/env bash ################################################################################ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ set -o pipefail source "$(dirname "$0")"/common.sh source "$(dirname "$0")"/common_docker.sh MAX_RETRY_SECONDS=120 IMAGE_BUILD_RETRIES=5 NODENAME=${NODENAME:-`hostname -f`} export MESOS_AGENT_CPU=1 echo "End-to-end directory $END_TO_END_DIR" start_time=$(date +%s) # make sure we stop our cluster at the end function cluster_shutdown { docker exec mesos-master bash -c "chmod -R ogu+rw $FLINK_LOG_DIR/ ${TEST_DATA_DIR}" docker-compose -f $END_TO_END_DIR/test-scripts/docker-mesos-cluster/docker-compose.yml down } on_exit cluster_shutdown function start_flink_cluster_with_mesos() { echo "Starting Flink on Mesos cluster" if ! retry_times $IMAGE_BUILD_RETRIES 0 build_image; then echo "ERROR: Could not build mesos image. Aborting..." exit 1 fi # build docker image with java and mesos build_image # we need to export the MVN_REPO location so that mesos can access the files referenced in HADOOP_CLASSPATH export MVN_REPO=`mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout` # start mesos cluster docker-compose -f $END_TO_END_DIR/test-scripts/docker-mesos-cluster/docker-compose.yml up -d # wait for the Mesos master and slave set up start_time=$(date +%s) wait_rest_endpoint_up "http://${NODENAME}:5050/slaves" "Mesos" "\{\"slaves\":\[.+\].*\}" # perform health checks containers_health_check "mesos-master" "mesos-slave" set_config_key "jobmanager.rpc.address" "mesos-master" set_config_key "rest.address" "mesos-master" docker exec --env HADOOP_CLASSPATH=$HADOOP_CLASSPATH -itd mesos-master bash -c "${FLINK_DIR}/bin/mesos-appmaster.sh -Dmesos.master=mesos-master:5050" wait_rest_endpoint_up "http://${NODENAME}:8081/taskmanagers" "Dispatcher" "\{\"taskmanagers\":\[.*\]\}" return 0 } function build_image() { echo "Building Mesos Docker container" docker build -f $END_TO_END_DIR/test-scripts/docker-mesos-cluster/Dockerfile \ -t flink/docker-mesos-cluster:latest \ $END_TO_END_DIR/test-scripts/docker-mesos-cluster/ } function wait_job_terminal_state_mesos { local job=$1 local expected_terminal_state=$2 wait_job_terminal_state $1 $2 "mesos-appmaster" } function wait_num_of_occurence_in_logs_mesos() { local text=$1 local number=$2 wait_num_of_occurence_in_logs $1 $2 "mesos-appmaster" }