[FLINK-14241][test] Add arm support for docker test

Docker-compose doesn't have arm release bin file. Use pip3 install
for arm arch

This closes apache#9782

flink-container/docker/build.sh

@@ -184,4 +184,13 @@ else
 
 fi
 
-docker build --build-arg flink_dist="${FLINK_DIST}" --build-arg job_artifacts="${JOB_ARTIFACTS_TARGET}" --build-arg hadoop_jar="${SHADED_HADOOP}" --build-arg python_version="${PYTHON_VERSION}" -t "${IMAGE_NAME}" .
+if [[ `uname -i` == 'aarch64' ]]; then
+ # openjdk:8-jre-alpine doesn't work on ARM. See bug: https://icedtea.classpath.org/bugzilla/show_bug.cgi?id=3740
+ cp Dockerfile Dockerfile-ARM
+ sed -i 's/openjdk:8-jre-alpine/arm64v8\/openjdk:8u201-jdk-alpine/' Dockerfile-ARM
+ DOCKERFILE="Dockerfile-ARM"
+else
+ DOCKERFILE="Dockerfile"
+fi
+
+docker build --build-arg flink_dist="${FLINK_DIST}" --build-arg job_artifacts="${JOB_ARTIFACTS_TARGET}" --build-arg hadoop_jar="${SHADED_HADOOP}" --build-arg python_version="${PYTHON_VERSION}" -t "${IMAGE_NAME}" -f $DOCKERFILE .

flink-end-to-end-tests/run-nightly-tests.sh

@@ -124,17 +124,19 @@ run_test "RocksDB Memory Management end-to-end test" "$END_TO_END_DIR/test-scrip
 if [[ ${PROFILE} != *"jdk11"* ]]; then
  run_test "Wordcount on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh dummy-fs"
 
- run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
- run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
-
  run_test "Run Kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh"
  run_test "Run kubernetes session test" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh"
 
  run_test "Running Flink over NAT end-to-end test" "$END_TO_END_DIR/test-scripts/test_nat.sh" "skip_check_exceptions"
 
- if [[ $PROFILE == *"include-hadoop"* ]]; then
- run_test "Run Mesos WordCount test" "$END_TO_END_DIR/test-scripts/test_mesos_wordcount.sh"
- run_test "Run Mesos multiple submission test" "$END_TO_END_DIR/test-scripts/test_mesos_multiple_submissions.sh"
+ if [[ `uname -i` != 'aarch64' ]]; then
+ # Hadoop YARN deosn't support aarch64 at this moment. See: https://issues.apache.org/jira/browse/HADOOP-16723
+ run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
+ run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
+ if [[ $PROFILE == *"include-hadoop"* ]]; then
+ run_test "Run Mesos WordCount test" "$END_TO_END_DIR/test-scripts/test_mesos_wordcount.sh"
+ run_test "Run Mesos multiple submission test" "$END_TO_END_DIR/test-scripts/test_mesos_multiple_submissions.sh"
+ fi
  fi
 
  run_test "Test PubSub connector with Docker based Google PubSub Emulator" "$END_TO_END_DIR/test-scripts/test_streaming_gcp_pubsub.sh"

flink-end-to-end-tests/test-scripts/common_kubernetes.sh

@@ -31,17 +31,22 @@ NON_LINUX_ENV_NOTE="****** Please start/stop minikube manually in non-linux envi
 
 # If running tests on non-linux os, the kubectl and minikube should be installed manually
 function setup_kubernetes_for_linux {
+ if [[ `uname -i` == 'aarch64' ]]; then
+ local arch='arm64'
+ else
+ local arch='amd64'
+ fi
  # Download kubectl, which is a requirement for using minikube.
  if ! [ -x "$(command -v kubectl)" ]; then
  echo "Installing kubectl ..."
  local version=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
- curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/amd64/kubectl && \
+ curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/$arch/kubectl && \
  chmod +x kubectl && sudo mv kubectl /usr/local/bin/
  fi
  # Download minikube.
  if ! [ -x "$(command -v minikube)" ]; then
  echo "Installing minikube ..."
- curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.2/minikube-linux-amd64 && \
+ curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.2/minikube-linux-$arch && \
  chmod +x minikube && sudo mv minikube /usr/local/bin/
  fi
 }