[CI] Factor out more Travis code and update GitHub Actions (ray-proje…

…ct#8085)

.travis.yml

@@ -121,8 +121,8 @@ matrix:
  - python setup.py check --restructuredtext --strict --metadata
  - cd ..
  # Run Bazel linter Buildifier.
- - wget -q https://dl.google.com/go/go1.12.linux-amd64.tar.gz
- - tar -xf go1.12.linux-amd64.tar.gz
+ - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz
+ - tar -xf go1.14.2.linux-amd64.tar.gz
  - mkdir $HOME/go_dir
  - export GOROOT=`pwd`/go
  - export GOPATH="$HOME/go_dir"
@@ -313,8 +313,8 @@ script:
 
  # ray operator tests
  - cd ./deploy/ray-operator/
- - ../../ci/suppress_output go build
- - ../../ci/suppress_output go test ./...
+ - CC=gcc ../../ci/suppress_output go build
+ - CC=gcc ../../ci/suppress_output go test ./...
  - cd ../..
 
  # random python tests TODO(ekl): these should be moved to bazel

build.sh

@@ -31,6 +31,8 @@ if [[ "$unamestr" == "Linux" ]]; then
  PARALLEL=1
 elif [[ "$unamestr" == "Darwin" ]]; then
  PARALLEL=$(sysctl -n hw.ncpu)
+elif [[ "${OSTYPE}" == "msys" ]]; then
+ PARALLEL="${NUMBER_OF_PROCESSORS-1}"
 else
  echo "Unrecognized platform."
  exit 1
@@ -106,8 +108,15 @@ echo "Using Python executable $PYTHON_EXECUTABLE."
 
 # Find the bazel executable. The script ci/travis/install-bazel.sh doesn't
 # always put the bazel executable on the PATH.
-BAZEL_EXECUTABLE=$(PATH="$PATH:$HOME/.bazel/bin" which bazel)
-echo "Using Bazel executable $BAZEL_EXECUTABLE."
+if [ -z "${BAZEL_EXECUTABLE-}" ]; then
+ BAZEL_EXECUTABLE=$(PATH="$PATH:$HOME/.bazel/bin" which bazel)
+fi
+if [ -f "${BAZEL_EXECUTABLE}" ]; then
+ echo "Using Bazel executable $BAZEL_EXECUTABLE."
+else
+ echo "Bazel not found: BAZEL_EXECUTABLE=\"${BAZEL_EXECUTABLE}\""
+ exit 1
+fi
 
 # Now we build everything.
 BUILD_DIR="$ROOT_DIR/build/"
@@ -130,7 +139,7 @@ popd
 
 
 if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then
- "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \
+ CC=gcc "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \
  --target="$ROOT_DIR/python/ray/thirdparty_files"
 fi
 

ci/travis/check_import_order.py

@@ -9,6 +9,7 @@
 """
 
 import argparse
+import io
 import re
 import sys
 from pathlib import Path
@@ -23,7 +24,7 @@ def check_import(file):
  "import setproctitle": -1
  }
 
- with open(file) as f:
+ with io.open(file, "r", encoding="utf-8") as f:
  for i, line in enumerate(f):
  for check in check_to_lines.keys():
  # This regex will match the following case

ci/travis/ci.sh

@@ -4,11 +4,19 @@
 
 unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us
 
-set -eo pipefail && if [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac
+set -eo pipefail && if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac
 
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
 WORKSPACE_DIR="${ROOT_DIR}/../.."
 
+suppress_output() {
+ "${WORKSPACE_DIR}"/ci/suppress_output "$@"
+}
+
+keep_alive() {
+ "${WORKSPACE_DIR}"/ci/keep_alive "$@"
+}
+
 # If provided the names of one or more environment variables, returns success if any of them is triggered.
 # Usage: should_run_job [VAR_NAME]...
 should_run_job() {
@@ -17,7 +25,7 @@ should_run_job() {
  local envvar active_triggers=()
  for envvar in "$@"; do
  if [ "${!envvar}" = 1 ]; then
- active_triggers+="${envvar}=${!envvar}" # success! we found at least one of the given triggers is occurring
+ active_triggers+=("${envvar}=${!envvar}") # success! we found at least one of the given triggers is occurring
  fi
  done
  if [ 0 -eq "${#active_triggers[@]}" ]; then
@@ -53,16 +61,33 @@ preload() {
  local variable_definitions
  variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py))
  if [ 0 -lt "${#variable_definitions[@]}" ]; then
- export "${variable_definitions[@]}"
+ local expression
+ expression="$(printf "%q " "${variable_definitions[@]}")"
+ eval "${expression}"
+ printf "%s\n" "${expression}" >> ~/.bashrc
  fi
 
  if ! (set +x && should_run_job ${job_names//,/ }); then
+ if [ -n "${GITHUB_WORKFLOW-}" ]; then
+ # If this job is to be skipped, emit an 'exit' command into .bashrc to quickly exit all following steps.
+ # This isn't needed for Travis (since everything runs in a single shell), but it is needed for GitHub Actions.
+ cat <<EOF1 >> ~/.bashrc
+ cat <<EOF2 1>&2
+Exiting shell as no triggers were active for this job:
+ ${job_names//,/}
+The active triggers during job initialization were the following:
+ ${variable_definitions[*]}
+EOF2
+ exit 0
+EOF1
+ fi
  exit 0
  fi
 }
 
 # Initializes the environment for the current job. Performs the following tasks:
-# - Calls 'exit 0' to quickly exit if provided a list of job names and none of them has been triggered.
+# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and
+# none of them has been triggered.
 # - Sets variables to indicate the job names that have been triggered.
 # Note: Please avoid exporting these variables. Instead, source any callees that need to use them.
 # This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u).
@@ -99,7 +124,7 @@ build() {
  fi
 
  if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then
- eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)"
+ eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)"
  fi
 
  if [ "${LINUX_WHEELS-}" = 1 ]; then
@@ -111,12 +136,12 @@ build() {
 
  # This command should be kept in sync with ray/python/README-building-wheels.md,
  # except the "${MOUNT_BAZEL_CACHE[@]}" part.
- "${WORKSPACE_DIR}"/ci/suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+ suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
  fi
 
  if [ "${MAC_WHEELS-}" = 1 ]; then
  # This command should be kept in sync with ray/python/README-building-wheels.md.
- "${WORKSPACE_DIR}"/ci/suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
+ suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
  fi
 }
 

ci/travis/install-toolchains.sh

@@ -32,6 +32,17 @@ install_toolchains() {
  install /dev/stdin "${target}"
  7z x -bsp0 -bso0 "${target}" -o"${targetdir}"
  rm -f -- "${target}"
+ (
+ # Add Clang/LLVM binaries to somewhere that's in already PATH
+ # (don't change PATH itself, to avoid invalidating Bazel's cache or having to manage environment variables)
+ mkdir -p -- ~/bin
+ set +x
+ local path
+ for path in "${targetdir}\\bin"/*.exe; do
+ local name="${path##*/}"
+ printf "%s\n" "#!/usr/bin/env bash" "exec \"${path}\" \"\$@\"" | install /dev/stdin ~/bin/"${name%.*}"
+ done
+ )
  else
  sudo tar -x -J --strip-components=1 -C "${targetdir}"
  command -V clang 1>&2

python/setup.py

@@ -14,15 +14,21 @@
 # before these files have been created, so we have to move the files
 # manually.
 
+exe_suffix = ".exe" if sys.platform == "win32" else ""
+
+# .pyd is the extension Python requires on Windows for shared libraries.
+# https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll
+pyd_suffix = ".pyd" if sys.platform == "win32" else ".so"
+
 # NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
 ray_files = [
  "ray/core/src/ray/thirdparty/redis/src/redis-server",
  "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
- "ray/core/src/plasma/plasma_store_server",
- "ray/_raylet.so",
- "ray/core/src/ray/raylet/raylet_monitor",
- "ray/core/src/ray/gcs/gcs_server",
- "ray/core/src/ray/raylet/raylet",
+ "ray/core/src/plasma/plasma_store_server" + exe_suffix,
+ "ray/_raylet" + pyd_suffix,
+ "ray/core/src/ray/raylet/raylet_monitor" + exe_suffix,
+ "ray/core/src/ray/gcs/gcs_server" + exe_suffix,
+ "ray/core/src/ray/raylet/raylet" + exe_suffix,
  "ray/streaming/_streaming.so",
 ]
 
@@ -103,6 +109,12 @@ def run(self):
  # that certain flags will not be passed along such as --user or sudo.
  # TODO(rkn): Fix this.
  command = ["../build.sh", "-p", sys.executable]
+ if sys.platform == "win32" and command[0].lower().endswith(".sh"):
+ # We can't run .sh files directly in Windows, so find a shell.
+ # Don't use "bash" instead of "sh", because that might run the Bash
+ # from WSL! (We want MSYS2's Bash, which is also sh by default.)
+ shell = os.getenv("BAZEL_SH", "sh") # NOT "bash"! (see above)
+ command.insert(0, shell)
  if build_java:
  # Also build binaries for Java if the above env variable exists.
  command += ["-l", "python,java"]