Various CI fixes and cleanup (ray-project#8289)

.bazelrc

@@ -2,7 +2,7 @@
 build --enable_platform_specific_config
 ###############################################################################
 # On Windows, provide: USE_CLANG_CL=1, CC=clang, BAZEL_LLVM, BAZEL_SH
-# On all platforms, provide: PYTHON2_BIN_PATH, PYTHON3_BIN_PATH
+# On all platforms, provide: PYTHON3_BIN_PATH=python
 ###############################################################################
 build --action_env=PATH
 build:linux --compilation_mode=opt

.github/workflows/main.yml

@@ -2,6 +2,12 @@ name: CI
 
 on: [push, pull_request]
 
+env:
+ # Git GITHUB_... variables are useful for translating Travis environment variables
+ GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ GITHUB_PULL_REQUEST: ${{ github.event.number }}
+
 jobs:
  build:
  name: ${{ matrix.name }}
@@ -18,7 +24,6 @@ jobs:
  - name: macos
  os: macos-10.15
  env:
- GITHUB_PULL_REQUEST: ${{ github.event.number }}
  PYTHON: 3.6
  TRAVIS_BUILD_DIR: ${{ github.workspace }}
  TRAVIS_COMMIT: ${{ github.sha }}

.travis.yml

@@ -2,6 +2,9 @@ language: generic
 # Use Ubuntu 16.04
 dist: xenial
 
+before_install:
+ - unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us
+
 matrix:
  include:
  - os: linux
@@ -10,9 +13,9 @@ matrix:
  - PYTHONWARNINGS=ignore
  - RAY_DEFAULT_BUILD=1
  - RAY_CYTHON_EXAMPLES=1
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
 
  - os: osx
@@ -22,9 +25,9 @@ matrix:
  - PYTHONWARNINGS=ignore
  - RAY_DEFAULT_BUILD=1
  - RAY_CYTHON_EXAMPLES=1
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
 
  - os: linux
@@ -33,9 +36,9 @@ matrix:
  - PYTHON=3.6 PYTHONWARNINGS=ignore
  - RAY_INSTALL_JAVA=1
  - RAY_GCS_ACTOR_SERVICE_ENABLED=true
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - ./java/test.sh
@@ -47,9 +50,9 @@ matrix:
  - RAY_INSTALL_JAVA=1
  - RAY_GCS_ACTOR_SERVICE_ENABLED=true
  - PYTHON=3.6 PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  # Streaming cpp test.
@@ -65,9 +68,9 @@ matrix:
  - RAY_INSTALL_JAVA=1
  - RAY_GCS_SERVICE_ENABLED=false
  - RAY_CYTHON_EXAMPLES=1
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
@@ -82,9 +85,9 @@ matrix:
  - RAY_INSTALL_JAVA=1
  - RAY_GCS_SERVICE_ENABLED=false
  - RAY_CYTHON_EXAMPLES=1
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...
@@ -93,12 +96,13 @@ matrix:
  env:
  - LINT=1
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init
  install:
+ - . ./ci/travis/ci.sh init
+ before_script:
+ - . ./ci/travis/ci.sh lint
  - . ./ci/travis/ci.sh build
  script:
- - . ./ci/travis/ci.sh lint
+ - true # we still need this block to exist, otherwise it will fall back to the global one
 
  # Build Linux wheels.
  - os: linux
@@ -107,9 +111,9 @@ matrix:
  - PYTHONWARNINGS=ignore
  - RAY_INSTALL_JAVA=1
  - RAY_GCS_ACTOR_SERVICE_ENABLED=true
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - . ./ci/travis/ci.sh test_wheels
@@ -123,9 +127,9 @@ matrix:
  - PYTHONWARNINGS=ignore
  - RAY_INSTALL_JAVA=1
  - RAY_GCS_ACTOR_SERVICE_ENABLED=true
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - . ./ci/travis/ci.sh test_wheels
@@ -140,9 +144,9 @@ matrix:
  - TORCH_VERSION=1.4
  - PYTHON=3.6
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_tf rllib/...
@@ -158,9 +162,9 @@ matrix:
  - TORCH_VERSION=1.4
  - PYTHON=3.6
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_tf rllib/...
@@ -175,9 +179,9 @@ matrix:
  - TORCH_VERSION=1.4
  - PYTHON=3.6
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_torch rllib/...
@@ -193,9 +197,9 @@ matrix:
  - TFP_VERSION=0.8
  - TORCH_VERSION=1.4
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/...
@@ -213,9 +217,9 @@ matrix:
  - TFP_VERSION=0.8
  - TORCH_VERSION=1.4
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/...
@@ -233,9 +237,9 @@ matrix:
  - TFP_VERSION=0.8
  - TORCH_VERSION=1.4
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/...
@@ -250,9 +254,9 @@ matrix:
  - TFP_VERSION=0.8
  - TORCH_VERSION=1.4
  - PYTHONWARNINGS=ignore
- before_install:
- - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
  install:
+ - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/...
@@ -262,9 +266,9 @@ matrix:
  env:
  - TESTSUITE=cpp_worker
  - PYTHON=3.6
- before_install:
- - . ./ci/travis/ci.sh init
  install:
+ - . ./ci/travis/ci.sh init
+ before_script:
  - . ./ci/travis/ci.sh build
  script:
  - . ./ci/travis/ci.sh test_cpp

BUILD.bazel

@@ -1526,11 +1526,12 @@ genrule(
  mv -f -- redis-cli.exe $(location redis-cli)
  """,
  "//conditions:default": """
- set -x &&
  tmpdir="redis.tmp" &&
  path=$(location @com_github_antirez_redis//:file) &&
  cp -p -L -R -- "$${path%/*}" "$${tmpdir}" &&
- make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" V=0 &&
+ chmod +x "$${tmpdir}"/deps/jemalloc/configure &&
+ parallel="$$(getconf _NPROCESSORS_ONLN || echo 1)"
+ make -s -C "$${tmpdir}" -j"$${parallel}" V=0 CFLAGS="$${CFLAGS-} -DLUA_USE_MKSTEMP -Wno-pragmas -Wno-empty-body" &&
  mv "$${tmpdir}"/src/redis-server $(location redis-server) &&
  chmod +x $(location redis-server) &&
  mv "$${tmpdir}"/src/redis-cli $(location redis-cli) &&

bazel/ray_deps_setup.bzl

@@ -82,6 +82,7 @@ def ray_deps_setup():
  "//thirdparty/patches:hiredis-windows-sigpipe.patch",
  "//thirdparty/patches:hiredis-windows-sockets.patch",
  "//thirdparty/patches:hiredis-windows-strerror.patch",
+ "//thirdparty/patches:redis-quiet.patch",
  ],
  )
 
@@ -214,8 +215,8 @@ def ray_deps_setup():
  url = "https://github.com/grpc/grpc/archive/4790ab6d97e634a1ede983be393f3bb3c132b2f7.tar.gz",
  sha256 = "df83bd8a08975870b8b254c34afbecc94c51a55198e6e3a5aab61d62f40b7274",
  patches = [
- "//thirdparty/patches:grpc-command-quoting.patch",
  "//thirdparty/patches:grpc-cython-copts.patch",
+ "//thirdparty/patches:grpc-python.patch",
  ],
  )
 

build.sh

@@ -133,7 +133,7 @@ git clone https://github.com/suquark/pickle5-backport
 pushd pickle5-backport
  git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
  CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel
- unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
+ unzip -q -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
 popd
 popd
 

ci/README.md

@@ -8,7 +8,7 @@ _Please double-check file/function/etc. names for changes, as this document may
 All dependencies (e.g. `apt`, `pip`) should be installed in `install_dependencies()`, following the same pattern as
 those that already exist.
 
-Once a dependency is added/removed, please ensure that if `reload_env` (or similar) is updated if it exists, as CI
+Once a dependency is added/removed, please ensure that shell environment variables are persisted appropriately, as CI
 systems differ on when `~/.bashrc` et al. are reloaded, if at all. (And they are not necessarily idempotent.)
 
 ### Bazel, environment variables, and caching
@@ -39,10 +39,11 @@ Adding new scripts has a number of pitfalls that easily take hours (even days) t
 
 The following practices can avoid such pitfalls while maintaining intuitive control flow:
 
-- Put all environment-modifying functions in the _same_ shell script, so that their invocation behaves intuitively.
+- Put all environment-modifying functions in the _top-level_ shell script, so that their invocation behaves intuitively.
  (The sheer length of the script is a secondary concern and can be mitigated by keeping functions modular.)
 
 - Avoid adding new scripts if possible. If it's necessary that you do so, call them instead of sourcing them.
+ Note that thies implies new scripts should not modify the environment, or the caller will not see such changes!
 
 - Always add code inside a function, not at global scope. Use `local` for variables where it makes sense.
  However, be careful and know the shell rules: for example, e.g. `local x=$(false)` succeeds even under `set -e`.