Merge branch 'master' into multi_gpu_other_algos

.flake8

@@ -20,4 +20,6 @@ ignore =
  W503
  W504
  W605
+ I
+ N
 avoid-escape = no

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+ # Tune/SGD/Doc requirements
+ - package-ecosystem: "pip"
+ # The requirements base directory currently only contains tune requirements.
+ # If we want to add more requirements here (Core, RLlib, etc.), then we should make subdirectories for each one.
+ directory: "/python/requirements"
+ schedule:
+ # Automatic upgrade checks Saturday at 12 AM.
+ # Dependabot updates can still be manually triggered via Github at any time.
+ interval: "weekly"
+ day: "saturday"
+ # 12 AM
+ time: "00:00"
+ # Use Pacific Standard Time
+ timezone: "America/Los_Angeles"
+ commit-message:
+ prefix: "[tune]"
+ include: "scope"
+ # Only 3 upgrade PRs open at a time.
+ open-pull-requests-limit: 3
+ reviewers:
+ - "ray-project/ray-tune"

.github/workflows/main.yml

@@ -1,6 +1,11 @@
 name: CI
 
-on: [push, pull_request]
+on:
+ push:
+ branches-ignore:
+ # Don't run CI for Dependabot branch pushes.
+ - "dependabot/**"
+ pull_request:
 
 env:
  # Git GITHUB_... variables are useful for translating Travis environment variables

.travis.yml

@@ -7,6 +7,10 @@ git:
  depth: false # Shallow clones can prevent diff against base branch
  quiet: true
 
+branches:
+ except:
+ - /dependabot.*/
+
 before_install:
  - unset -f cd # Travis defines this on Mac for RVM, but it breaks the Mac build
  - |
@@ -20,50 +24,6 @@ before_install:
 
 matrix:
  include:
- - os: linux
- env:
- - PYTHON=3.6 SMALL_AND_LARGE_TESTS=1 RAY_ENABLE_NEW_SCHEDULER=1
- - PYTHONWARNINGS=ignore
- - RAY_DEFAULT_BUILD=1
- - RAY_CYTHON_EXAMPLES=1
- - RAY_USE_RANDOM_PORTS=1
- install:
- - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED,RAY_CI_DASHBOARD_AFFECTED
- before_script:
- - . ./ci/travis/ci.sh build
- script:
- # bazel python tests. This should be run last to keep its logs at the end of travis logs.
- - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-new_scheduler_broken python/ray/tests/...; fi
-
- - os: linux
- env:
- - PYTHON=3.6 MEDIUM_TESTS_A_TO_J=1 RAY_ENABLE_NEW_SCHEDULER=1
- - PYTHONWARNINGS=ignore
- - RAY_DEFAULT_BUILD=1
- - RAY_CYTHON_EXAMPLES=1
- - RAY_USE_RANDOM_PORTS=1
- install:
- - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED,RAY_CI_DASHBOARD_AFFECTED
- before_script:
- - . ./ci/travis/ci.sh build
- script:
- # bazel python tests for medium size tests. Used for parallelization.
- - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,medium_size_python_tests_a_to_j,-new_scheduler_broken python/ray/tests/...; fi
-
- - os: linux
- env:
- - PYTHON=3.6 MEDIUM_TESTS_K_TO_Z=1 RAY_ENABLE_NEW_SCHEDULER=1
- - PYTHONWARNINGS=ignore
- - RAY_DEFAULT_BUILD=1
- - RAY_CYTHON_EXAMPLES=1
- - RAY_USE_RANDOM_PORTS=1
- install:
- - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED,RAY_CI_DASHBOARD_AFFECTED
- before_script:
- - . ./ci/travis/ci.sh build
- script:
- # bazel python tests for medium size tests. Used for parallelization.
- - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,medium_size_python_tests_k_to_z,-new_scheduler_broken python/ray/tests/...; fi
  - os: linux
  env:
  - PYTHON=3.6 SMALL_AND_LARGE_TESTS=1
@@ -90,6 +50,7 @@ matrix:
  script:
  # bazel python tests for medium size tests. Used for parallelization.
  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,medium_size_python_tests_a_to_j python/ray/tests/...; fi
+ - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,client_tests --test_env=RAY_CLIENT_MODE=1 python/ray/tests/...; fi
 
  - os: linux
  env:
@@ -184,6 +145,7 @@ matrix:
  - RAY_INSTALL_JAVA=1
  - PYTHON=3.6 PYTHONWARNINGS=ignore
  - RAY_USE_RANDOM_PORTS=1
+ - RAY_ENABLE_NEW_SCHEDULER=0
  language: java
  jdk: openjdk8
  install:
@@ -193,8 +155,9 @@ matrix:
  script:
  # Streaming cpp test.
  - if [ $RAY_CI_STREAMING_CPP_AFFECTED == "1" ]; then bazel test --config=ci $(./scripts/bazel_export_options) --test_output=all //streaming:all && bash streaming/src/test/run_streaming_queue_test.sh; fi
- - if [ $RAY_CI_STREAMING_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 streaming/python/tests/; fi
- - if [ $RAY_CI_STREAMING_JAVA_AFFECTED == "1" ]; then ./streaming/java/test.sh; fi
+ # BROKEN
+ # - if [ $RAY_CI_STREAMING_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 streaming/python/tests/; fi
+ # - if [ $RAY_CI_STREAMING_JAVA_AFFECTED == "1" ]; then ./streaming/java/test.sh; fi
 
  - os: linux
  env:
@@ -219,7 +182,7 @@ matrix:
  - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
  before_script:
  - brew tap adoptopenjdk/openjdk
- - brew cask install adoptopenjdk8
+ - brew install --cask adoptopenjdk8
  - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
  - java -version
  - . ./ci/travis/ci.sh build
@@ -399,9 +362,6 @@ matrix:
  env:
  - TUNE_TESTING=1
  - PYTHON=3.6
- - TF_VERSION=2.2.0
- - TFP_VERSION=0.8
- - TORCH_VERSION=1.7
  - PYTHONWARNINGS=ignore
  install:
  - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED
@@ -419,9 +379,6 @@ matrix:
  env:
  - SGD_TESTING=1
  - PYTHON=3.6
- - TF_VERSION=2.1.0
- - TFP_VERSION=0.8
- - TORCH_VERSION=1.7
  - PYTHONWARNINGS=ignore
  install:
  - . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED
@@ -439,9 +396,6 @@ matrix:
  env:
  - DOC_TESTING=1
  - PYTHON=3.6
- - TF_VERSION=2.1.0
- - TFP_VERSION=0.8
- - TORCH_VERSION=1.7
  - PYTHONWARNINGS=ignore
  install:
  - . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_DOC_AFFECTED
@@ -457,9 +411,6 @@ matrix:
  env:
  - PYTHON=3.7 TUNE_TESTING=1
  - INSTALL_HOROVOD=1
- - TF_VERSION=2.1.0
- - TFP_VERSION=0.8
- - TORCH_VERSION=1.7
  - PYTHONWARNINGS=ignore
  install:
  - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED
@@ -577,4 +528,3 @@ deploy:
  repo: ray-project/ray
  branch: master
  condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1 || $LINUX_JARS = 1
-

BUILD.bazel

@@ -248,7 +248,6 @@ cc_library(
  "src/ray/object_manager/plasma/client.h",
  "src/ray/object_manager/plasma/common.h",
  "src/ray/object_manager/plasma/compat.h",
- "src/ray/object_manager/plasma/external_store.h",
  "src/ray/object_manager/plasma/connection.h",
  "src/ray/object_manager/plasma/malloc.h",
  "src/ray/object_manager/plasma/plasma.h",
@@ -275,7 +274,6 @@ cc_library(
  ":ray_common",
  ":ray_util",
  "//src/ray/protobuf:common_cc_proto",
- "@arrow",
  "@com_github_google_glog//:glog",
  "@msgpack",
  ],
@@ -287,7 +285,6 @@ cc_library(
  "src/ray/object_manager/plasma/create_request_queue.cc",
  "src/ray/object_manager/plasma/dlmalloc.cc",
  "src/ray/object_manager/plasma/eviction_policy.cc",
- "src/ray/object_manager/plasma/external_store.cc",
  "src/ray/object_manager/plasma/plasma_allocator.cc",
  "src/ray/object_manager/plasma/quota_aware_policy.cc",
  "src/ray/object_manager/plasma/store.cc",
@@ -297,7 +294,6 @@ cc_library(
  "src/ray/object_manager/common.h",
  "src/ray/object_manager/plasma/create_request_queue.h",
  "src/ray/object_manager/plasma/eviction_policy.h",
- "src/ray/object_manager/plasma/external_store.h",
  "src/ray/object_manager/plasma/plasma_allocator.h",
  "src/ray/object_manager/plasma/quota_aware_policy.h",
  "src/ray/object_manager/plasma/store.h",
@@ -363,7 +359,6 @@ cc_library(
  ":ray_util",
  "//src/ray/protobuf:common_cc_proto",
  "//src/ray/protobuf:gcs_cc_proto",
- "@arrow",
  "@boost//:asio",
  "@com_github_grpc_grpc//:grpc++",
  "@com_google_absl//absl/container:flat_hash_map",
@@ -777,6 +772,16 @@ cc_test(
  ],
 )
 
+cc_test(
+ name = "lease_policy_test",
+ srcs = ["src/ray/core_worker/test/lease_policy_test.cc"],
+ copts = COPTS,
+ deps = [
+ ":core_worker_lib",
+ "@com_google_googletest//:gtest_main",
+ ],
+)
+
 cc_test(
  name = "cluster_resource_scheduler_test",
  srcs = [
@@ -880,7 +885,7 @@ cc_test(
 )
 
 cc_test(
- name = "local_placement_group_manager_test",
+ name = "placement_group_resource_manager_test",
  srcs = ["src/ray/raylet/placement_group_resource_manager_test.cc"],
  copts = COPTS,
  deps = [
@@ -956,8 +961,8 @@ cc_test(
 )
 
 cc_test(
- name = "task_dependency_manager_test",
- srcs = ["src/ray/raylet/task_dependency_manager_test.cc"],
+ name = "dependency_manager_test",
+ srcs = ["src/ray/raylet/dependency_manager_test.cc"],
  copts = COPTS,
  deps = [
  ":raylet_lib",
@@ -1020,7 +1025,6 @@ cc_test(
 cc_library(
  name = "gcs_test_util_lib",
  hdrs = [
- "src/ray/gcs/test/accessor_test_base.h",
  "src/ray/gcs/test/gcs_test_util.h",
  ],
  copts = COPTS,
@@ -1621,111 +1625,6 @@ cc_library(
  ],
 )
 
-# TODO(micafan) Support test group in future. Use test group we can run all gcs test once.
-cc_test(
- name = "redis_gcs_client_test",
- srcs = ["src/ray/gcs/test/redis_gcs_client_test.cc"],
- args = [
- "$(location redis-server)",
- "$(location redis-cli)",
- "$(location libray_redis_module.so)",
- ],
- copts = COPTS,
- data = [
- "//:libray_redis_module.so",
- "//:redis-cli",
- "//:redis-server",
- ],
- deps = [
- ":gcs",
- "@com_google_googletest//:gtest_main",
- ],
-)
-
-cc_test(
- name = "redis_actor_info_accessor_test",
- srcs = ["src/ray/gcs/test/redis_actor_info_accessor_test.cc"],
- args = [
- "$(location redis-server)",
- "$(location redis-cli)",
- "$(location libray_redis_module.so)",
- ],
- copts = COPTS,
- data = [
- "//:libray_redis_module.so",
- "//:redis-cli",
- "//:redis-server",
- ],
- deps = [
- ":gcs",
- ":gcs_test_util_lib",
- "@com_google_googletest//:gtest_main",
- ],
-)
-
-cc_test(
- name = "redis_object_info_accessor_test",
- srcs = ["src/ray/gcs/test/redis_object_info_accessor_test.cc"],
- args = [
- "$(location redis-server)",
- "$(location redis-cli)",
- "$(location libray_redis_module.so)",
- ],
- copts = COPTS,
- data = [
- "//:libray_redis_module.so",
- "//:redis-cli",
- "//:redis-server",
- ],
- deps = [
- ":gcs",
- ":gcs_test_util_lib",
- "@com_google_googletest//:gtest_main",
- ],
-)
-
-cc_test(
- name = "redis_job_info_accessor_test",
- srcs = ["src/ray/gcs/test/redis_job_info_accessor_test.cc"],
- args = [
- "$(location redis-server)",
- "$(location redis-cli)",
- "$(location libray_redis_module.so)",
- ],
- copts = COPTS,
- data = [
- "//:libray_redis_module.so",
- "//:redis-cli",
- "//:redis-server",
- ],
- deps = [
- ":gcs",
- ":gcs_test_util_lib",
- "@com_google_googletest//:gtest_main",
- ],
-)
-
-cc_test(
- name = "redis_node_info_accessor_test",
- srcs = ["src/ray/gcs/test/redis_node_info_accessor_test.cc"],
- args = [
- "$(location redis-server)",
- "$(location redis-cli)",
- "$(location libray_redis_module.so)",
- ],
- copts = COPTS,
- data = [
- "//:libray_redis_module.so",
- "//:redis-cli",
- "//:redis-server",
- ],
- deps = [
- ":gcs",
- ":gcs_test_util_lib",
- "@com_google_googletest//:gtest_main",
- ],
-)
-
 cc_test(
  name = "asio_test",
  srcs = ["src/ray/gcs/test/asio_test.cc"],