Merge remote-tracking branch 'upstream/master'

Signed-off-by: Balaji Veeramani <[email protected]>

.bazelrc

@@ -29,6 +29,9 @@ test --test_env=PYENV_VERSION
 test --test_env=PYENV_SHELL
 # Do not send usage stats to the server for tests
 test --test_env=RAY_USAGE_STATS_REPORT_URL="http:https://127.0.0.1:8000"
+# Enable cluster mode for OSX and Windows. By default, Ray
+# will not allow multinode OSX and Windows clusters.
+test --test_env=RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER="1"
 # This is needed for some core tests to run correctly
 build:windows --enable_runfiles
 # TODO(mehrdadn): Revert the "-\\.(asm|S)$" exclusion when this Bazel bug

.buildkite/pipeline.arm64.yml

@@ -74,7 +74,7 @@
  - pip install -q docker aws_requests_auth boto3
  - ./ci/env/env_info.sh
  - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
- - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cu113 cu116 --build-type BUILDKITE --build-base --suffix aarch64
+ - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cu113 cu116 cu118 --build-type BUILDKITE --build-base --suffix aarch64
 
 - label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (1/2)"
  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]

.buildkite/pipeline.build.yml

@@ -102,7 +102,7 @@
  - pip install -q docker aws_requests_auth boto3
  - ./ci/env/env_info.sh
  - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
- - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cu111 cu112 cu113 cu116 --build-type BUILDKITE --build-base
+ - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cu111 cu112 cu113 cu116 cu118 --build-type BUILDKITE --build-base
 
 - label: ":docker: Build Images: py38 (1/2)"
  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
@@ -574,6 +574,10 @@
  - pushd /ray && git clean -f -f -x -d -e .whl -e python/ray/dashboard/client && popd
  - bazel clean --expunge
  - export WANDB_MODE=offline
+ # Horovod needs to be installed separately (needed for API ref imports)
+ - ./ci/env/install-horovod.sh
+ # See https://stackoverflow.com/questions/63383400/error-cannot-uninstall-ruamel-yaml-while-creating-docker-image-for-azure-ml-a
+ - pip install mosaicml==0.10.1 --ignore-installed
  - ./ci/ci.sh build
 
 - label: ":octopus: Tune multinode tests"

.buildkite/pipeline.macos.yml

@@ -46,6 +46,7 @@ steps:
  - export MAC_WHEELS=1
  - export MAC_JARS=1
  - export RAY_INSTALL_JAVA=1
+ - export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1
  - . ./ci/ci.sh init && source ~/.zshenv
  - ./ci/ci.sh build
  # Test wheels

.buildkite/pipeline.ml.yml

@@ -297,11 +297,22 @@
  - sudo service mongodb stop
  - sudo apt-get purge -y mongodb*
 
+- label: "Dataset tests (streaming executor)"
+ conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_DATA_AFFECTED"]
+ instance_size: medium
+ commands:
+ - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh
+ - ./ci/env/env_info.sh
+ - bazel test --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATASET_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
+ - bazel test --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATASET_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=ray_data python/ray/air/...
+
 - label: "Dataset tests (Arrow nightly)"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"]
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - ./ci/env/install-java.sh
  - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -312,6 +323,7 @@
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - ./ci/env/install-java.sh
  - DATA_PROCESSING_TESTING=1 ARROW_VERSION=10.* ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -322,6 +334,7 @@
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - ./ci/env/install-java.sh
  - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -332,6 +345,7 @@
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - ./ci/env/install-java.sh
  - DATA_PROCESSING_TESTING=1 ARROW_VERSION=8.* ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -342,6 +356,7 @@
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - ./ci/env/install-java.sh
  - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -352,6 +367,7 @@
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+ - ./ci/env/install-java.sh
  - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...

.buildkite/pipeline.windows.yml

@@ -17,6 +17,7 @@ prelude_commands: &prelude_commands |-
  export BUILD="1"
  export DL="1"
  powershell ci/pipeline/fix-windows-container-networking.ps1
+ powershell ci/pipeline/fix-windows-bazel.ps1
  cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 
 upload_wheels_if_needed: &upload_wheels_if_needed |-
@@ -35,6 +36,7 @@ steps:
  - conda init
  - . ./ci/ci.sh init
  - ./ci/ci.sh build
+ - export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER="1"
  - if [ "${BUILDKITE_PARALLEL_JOB}" = "0" ]; then ./ci/ci.sh test_core; fi
  # The next command will be sharded into $parallelism shards.
  - ./ci/ci.sh test_python

.github/CODEOWNERS

@@ -127,6 +127,6 @@
 # Buildkite pipeline management
 .buildkite/hooks @simon-mo @krfricke
 
-/release/ray_release @krfricke @rkooo567 @simon-mo
+/release/ray_release @krfricke @rkooo567 @simon-mo @Yard1
 
 /.github/ISSUE_TEMPLATE/ @ericl @stephanie-wang @scv119 @pcmoritz

README.rst

@@ -35,6 +35,8 @@ Or more about `Ray Core`_ and its key abstractions:
 - `Actors`_: Stateful worker processes created in the cluster.
 - `Objects`_: Immutable values accessible across the cluster.
 
+Monitor and debug Ray applications and clusters using the `Ray dashboard <https://docs.ray.io/en/latest/ray-core/ray-dashboard.html>`__.
+
 Ray runs on any machine, cluster, cloud provider, and Kubernetes, and features a growing
 `ecosystem of community integrations`_.
 

WORKSPACE

@@ -22,7 +22,7 @@ load("@bazel_skylib//lib:versions.bzl", "versions")
 
 # When the bazel version is updated, make sure to update it
 # in setup.py as well.
-versions.check(minimum_bazel_version = "4.2.1")
+versions.check(minimum_bazel_version = "5.4.0")
 
 # Tools to generate `compile_commands.json` to enable awesome tooling of the C language family.
 # Just run `bazel run @hedron_compile_commands//:refresh_all`

bazel/ray_deps_setup.bzl

@@ -156,21 +156,15 @@ def ray_deps_setup():
  # declaring it here allows us to avoid patching the latter.
  name = "boost",
  build_file = "@com_github_nelhage_rules_boost//:BUILD.boost",
- sha256 = "83bfc1507731a0906e387fc28b7ef5417d591429e51e788417fe9ff025e116b1",
- url = "https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source/boost_1_74_0.tar.bz2",
- patches = [
- "@com_github_ray_project_ray//thirdparty/patches:boost-exception-no_warn_typeid_evaluated.patch",
- ],
+ sha256 = "71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa",
+ url = "https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2",
  )
 
  auto_http_archive(
  name = "com_github_nelhage_rules_boost",
  # If you update the Boost version, remember to update the 'boost' rule.
- url = "https://github.com/nelhage/rules_boost/archive/652b21e35e4eeed5579e696da0facbe8dba52b1f.tar.gz",
- sha256 = "c1b8b2adc3b4201683cf94dda7eef3fc0f4f4c0ea5caa3ed3feffe07e1fb5b15",
- patches = [
- "@com_github_ray_project_ray//thirdparty/patches:rules_boost-windows-linkopts.patch",
- ],
+ url = "https://github.com/nelhage/rules_boost/archive/57c99395e15720e287471d79178d36a85b64d6f6.tar.gz",
+ sha256 = "490d11425393eed068966a4990ead1ff07c658f823fd982fddac67006ccc44ab",
  )
 
  auto_http_archive(
@@ -206,12 +200,13 @@ def ray_deps_setup():
 
  auto_http_archive(
  name = "io_opencensus_cpp",
- url = "https://github.com/census-instrumentation/opencensus-cpp/archive/b14a5c0dcc2da8a7fc438fab637845c73438b703.zip",
- sha256 = "6592e07672e7f7980687f6c1abda81974d8d379e273fea3b54b6c4d855489b9d",
+ url = "https://github.com/census-instrumentation/opencensus-cpp/archive/5e5f2632c84e2230fb7ccb8e336f603d2ec6aa1b.zip",
+ sha256 = "1b88d6663f05c6a56c1604eb2afad22831d5f28a76f6fab8f37187f1e4ace425",
  patches = [
  "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-harvest-interval.patch",
  "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-shutdown-api.patch",
  ],
+ patch_args = ["-p1"], 
  )
 
  # OpenCensus depends on Abseil so we have to explicitly pull it in.

ci/build/build-docker-images.py

@@ -67,7 +67,7 @@
 # If changing the CUDA version in the below line, you should also change the base Docker
 # image being used in ~/ci/docker/Dockerfile.base.gpu to match the same image being used
 # here.
-ML_CUDA_VERSION = "cu118"
+ML_CUDA_VERSION = "cu116"
 
 DEFAULT_PYTHON_VERSION = "py37"
 

ci/docker/base.gpu.Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04
+FROM nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST

ci/docker/gpu.Dockerfile

@@ -23,7 +23,7 @@ COPY . .
 
 RUN env
 
+RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
+
 # Install Ray
 RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=0 bash --login -i -c -- "python3 -m pip install -e /ray/python/"
-
-RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh

ci/docker/ml.Dockerfile

@@ -12,7 +12,7 @@ WORKDIR /ray
 # Copy new ray files
 COPY . .
 
+RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
+
 # Install Ray
 RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=0 bash --login -i -c -- "python3 -m pip install -e /ray/python/"
-
-RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh

ci/docker/test.Dockerfile

@@ -9,10 +9,10 @@ RUN rm -rf /ray
 RUN mkdir /ray
 WORKDIR /ray
 
-# Copy new ray files
+# Copy new ray files.
 COPY . .
 
+RUN bash --login -i ./ci/env/install-dependencies.sh
+
 # Install Ray
 RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=0 bash --login -i -c -- "python3 -m pip install -e /ray/python/"
-
-RUN bash --login -i ./ci/env/install-dependencies.sh

ci/env/install-java.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Push caller's shell options (quietly)
+{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null
+
+set -euxo pipefail
+
+SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+WORKSPACE_DIR="${SCRIPT_DIR}/../.."
+
+sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk
+"${WORKSPACE_DIR}"/java/build-jar-multiplatform.sh linux
+
+# Pop caller's shell options (quietly)
+{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null

ci/pipeline/fix-windows-bazel.ps1

@@ -0,0 +1,13 @@
+$Env:BAZEL_URL="https://github.com/bazelbuild/bazel/releases/download/5.4.0/bazel-5.4.0-windows-x86_64.zip"
+Write-Host ('Downloading {0} ...' -f $env:BAZEL_URL);
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+Invoke-WebRequest -Uri $env:BAZEL_URL -OutFile 'bazel.zip';
+Write-Host 'Expanding ...';
+New-Item -ItemType Directory -Path C:\bazel -Force | Out-Null;
+Expand-Archive bazel.zip -DestinationPath C:\bazel -Force;
+Write-Host 'Removing ...';
+Remove-Item bazel.zip -Force;
+[Environment]::SetEnvironmentVariable("BAZEL_PATH", "C:\bazel\bazel.exe", [System.EnvironmentVariableTarget]::Machine)
+$systemPath = [Environment]::GetEnvironmentVariable('Path', [System.EnvironmentVariableTarget]::Machine)
+$systemPath += ';C:\bazel'
+[Environment]::SetEnvironmentVariable("PATH", $systemPath, [System.EnvironmentVariableTarget]::Machine)

cpp/BUILD.bazel

@@ -23,7 +23,6 @@ cc_binary(
  deps = [
  ":ray_api",
  ":symbols/ray_api_exported_symbols_linux.lds",
- ":symbols/ray_api_exported_symbols_mac.lds",
  ],
 )
 

cpp/src/ray/config_internal.cc

@@ -84,6 +84,12 @@ ABSL_FLAG(int,
  -1,
  "The computed hash of the runtime env for this worker.");
 
+ABSL_FLAG(std::string,
+ ray_job_namespace,
+ "",
+ "The namespace of job. If not set,"
+ " a unique value will be randomly generated.");
+
 using json = nlohmann::json;
 
 namespace ray {
@@ -132,6 +138,8 @@ void ConfigInternal::Init(RayConfig &config, int argc, char **argv) {
 
  if (!FLAGS_ray_code_search_path.CurrentValue().empty()) {
  // Code search path like this "/path1/xxx.so:/path2".
+ RAY_LOG(DEBUG) << "The code search path is "
+ << FLAGS_ray_code_search_path.CurrentValue();
  code_search_path = absl::StrSplit(
  FLAGS_ray_code_search_path.CurrentValue(), ':', absl::SkipEmpty());
  }
@@ -205,8 +213,13 @@ void ConfigInternal::Init(RayConfig &config, int argc, char **argv) {
  }
  }
  if (worker_type == WorkerType::DRIVER) {
- ray_namespace =
- config.ray_namespace.empty() ? GenerateUUIDV4() : config.ray_namespace;
+ ray_namespace = config.ray_namespace;
+ if (!FLAGS_ray_job_namespace.CurrentValue().empty()) {
+ ray_namespace = FLAGS_ray_job_namespace.CurrentValue();
+ }
+ if (ray_namespace.empty()) {
+ ray_namespace = GenerateUUIDV4();
+ }
  }
 
  auto job_config_json_string = std::getenv("RAY_JOB_CONFIG_JSON_ENV_VAR");

cpp/src/ray/runtime/object/native_object_store.cc

@@ -101,15 +101,31 @@ std::vector<std::shared_ptr<msgpack::sbuffer>> NativeObjectStore::GetRaw(
  for (size_t i = 0; i < results.size(); i++) {
  const auto &meta = results[i]->GetMetadata();
  const auto &data_buffer = results[i]->GetData();
+ std::string meta_str = "";
  if (meta != nullptr) {
- std::string meta_str((char *)meta->Data(), meta->Size());
+ meta_str = std::string((char *)meta->Data(), meta->Size());
  CheckException(meta_str, data_buffer);
  }
 
- auto sbuffer = std::make_shared<msgpack::sbuffer>(data_buffer->Size());
- sbuffer->write(reinterpret_cast<const char *>(data_buffer->Data()),
- data_buffer->Size());
- result_sbuffers.push_back(sbuffer);
+ const char *data = nullptr;
+ size_t data_size = 0;
+ if (data_buffer) {
+ data = reinterpret_cast<const char *>(data_buffer->Data());
+ data_size = data_buffer->Size();
+ }
+ if (meta_str == METADATA_STR_RAW) {
+ // TODO(LarryLian) In order to minimize the modification,
+ // there is an extra serialization here, but the performance will be a little worse.
+ // This code can be optimized later to improve performance
+ auto raw_buffer = Serializer::Serialize(data, data_size);
+ auto sbuffer = std::make_shared<msgpack::sbuffer>(raw_buffer.size());
+ sbuffer->write(raw_buffer.data(), raw_buffer.size());
+ result_sbuffers.push_back(sbuffer);
+ } else {
+ auto sbuffer = std::make_shared<msgpack::sbuffer>(data_size);
+ sbuffer->write(data, data_size);
+ result_sbuffers.push_back(sbuffer);
+ }
  }
  return result_sbuffers;
 }

cpp/src/ray/runtime/task/task_executor.cc

@@ -156,12 +156,30 @@ Status TaskExecutor::ExecuteTask(
  ArgsBufferList ray_args_buffer;
  for (size_t i = 0; i < args_buffer.size(); i++) {
  auto &arg = args_buffer.at(i);
+ std::string meta_str = "";
+ if (arg->GetMetadata() != nullptr) {
+ meta_str = std::string((const char *)arg->GetMetadata()->Data(),
+ arg->GetMetadata()->Size());
+ }
  msgpack::sbuffer sbuf;
- if (cross_lang) {
- sbuf.write((const char *)(arg->GetData()->Data()) + XLANG_HEADER_LEN,
- arg->GetData()->Size() - XLANG_HEADER_LEN);
+ const char *arg_data = nullptr;
+ size_t arg_data_size = 0;
+ if (arg->GetData()) {
+ arg_data = reinterpret_cast<const char *>(arg->GetData()->Data());
+ arg_data_size = arg->GetData()->Size();
+ }
+ if (meta_str == METADATA_STR_RAW) {
+ // TODO(LarryLian) In order to minimize the modification,
+ // there is an extra serialization here, but the performance will be a little worse.
+ // This code can be optimized later to improve performance
+ const auto &raw_buffer = Serializer::Serialize(arg_data, arg_data_size);
+ sbuf.write(raw_buffer.data(), raw_buffer.size());
+ } else if (cross_lang) {
+ RAY_CHECK(arg_data != nullptr)
+ << "Task " << task_name << " no." << i << " arg data is null.";
+ sbuf.write(arg_data + XLANG_HEADER_LEN, arg_data_size - XLANG_HEADER_LEN);
  } else {
- sbuf.write((const char *)(arg->GetData()->Data()), arg->GetData()->Size());
+ sbuf.write(arg_data, arg_data_size);
  }
 
  ray_args_buffer.push_back(std::move(sbuf));